# -*- coding: utf-8 -*-
"""

    @author: Fabio Erculiani <lxnay@sabayon.org>
    @contact: lxnay@sabayon.org
    @copyright: Fabio Erculiani
    @license: GPL-2

    B{Entropy Package Manager Server Main Interfaces}.

"""
import codecs
import collections
import copy
import errno
import hashlib
import os
import re
import shutil
import stat
import subprocess
import sys
import time
import threading

from entropy.exceptions import OnlineMirrorError, PermissionDenied, \
    SystemDatabaseError, RepositoryError
from entropy.const import etpConst, etpSys, const_setup_perms, \
    const_create_working_dirs, const_convert_to_unicode, \
    const_setup_file, const_get_stringtype, const_debug_write, \
    const_debug_enabled, const_convert_to_rawstring, const_mkdtemp, \
    const_mkstemp, const_file_readable
from entropy.output import purple, red, darkgreen, \
    bold, brown, blue, darkred, teal
from entropy.cache import EntropyCacher
from entropy.server.interfaces.mirrors import Server as MirrorsServer
from entropy.i18n import _
from entropy.core import BaseConfigParser
from entropy.core.settings.base import SystemSettings
from entropy.core.settings.plugins.skel import SystemSettingsPlugin
from entropy.transceivers import EntropyTransceiver
from entropy.db import EntropyRepository
from entropy.db.skel import EntropyRepositoryPlugin
from entropy.server.interfaces.db import ServerRepositoryStatus, \
    ServerPackagesRepository
from entropy.spm.plugins.factory import get_default_instance as get_spm, \
    get_default_class as get_spm_class
from entropy.qa import QAInterfacePlugin
from entropy.security import Repository as RepositorySecurity
from entropy.db.exceptions import ProgrammingError
from entropy.client.interfaces import Client
from entropy.client.interfaces.db import InstalledPackagesRepository, \
    GenericRepository
from entropy.client.misc import ConfigurationUpdates, ConfigurationFiles

import entropy.dep
import entropy.tools
import entropy.dump

SERVER_QA_PLUGIN = "ServerQAInterfacePlugin"


class ServerEntropyRepositoryPlugin(EntropyRepositoryPlugin):

    PLUGIN_ID = "__server__"

    def __init__(self, server_interface, metadata = None):
        """
        Entropy server-side repository ServerPackagesRepository Plugin class.
        This class will be instantiated and automatically added to
        ServerPackagesRepository instances generated by Entropy Server.

        @param server_interface: Entropy Server interface instance
        @type server_interface: entropy.server.interfaces.Server class
        @param metadata: any dict form metadata map (key => value)
        @type metadata: dict
        """
        EntropyRepositoryPlugin.__init__(self)
        self._cacher = EntropyCacher()
        self._settings = SystemSettings()
        self.srv_sys_settings_plugin = \
            etpConst['system_settings_plugins_ids']['server_plugin']
        self._server = server_interface
        if metadata is None:
            self._metadata = {}
        else:
            self._metadata = metadata

    def get_id(self):
        return ServerEntropyRepositoryPlugin.PLUGIN_ID

    def get_metadata(self):
        """
        This method should always return a direct reference to the object and
        NOT a copy.
        """
        return self._metadata

    def add_plugin_hook(self, entropy_repository_instance):
        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
                self,)
            )

        repo = entropy_repository_instance.repository_id()
        local_dbfile = self._metadata['local_dbfile']
        if local_dbfile is not None:
            taint_file = self._server._get_local_repository_taint_file(
                repo)
            if os.path.isfile(taint_file):
                dbs = ServerRepositoryStatus()
                dbs.set_tainted(local_dbfile)
                dbs.set_bumped(local_dbfile)

        if "__temporary__" in self._metadata: # in-memory db?
            local_dbfile_exists = True
        else:
            local_dbfile_exists = os.path.lexists(local_dbfile)

        if not local_dbfile_exists:
            # better than having a completely broken db
            self._metadata['read_only'] = False
            # force parameters, only ServerEntropyRepository exposes
            # the setReadonly method
            entropy_repository_instance.setReadonly(False)
            entropy_repository_instance.initializeRepository()
            entropy_repository_instance.commit()

        out_intf = self._metadata.get('output_interface')
        if out_intf is not None:
            entropy_repository_instance.output = out_intf.output
            entropy_repository_instance.ask_question = out_intf.ask_question

        return 0

    def close_repo_hook(self, entropy_repository_instance):
        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling close_repo_hook => %s" % (
                self,)
            )

        # this happens because close_repositories() might be called
        # before _setup_services() and in general, at any time, so, in this
        # case, there is no need to print bullshit to dev.
        if self._server.Mirrors is None:
            return 0

        repo = entropy_repository_instance.repository_id()
        dbfile = self._metadata['local_dbfile']
        if dbfile is None:
            # fake repo, or temporary one
            return 0

        read_only = self._metadata['read_only']
        if not read_only:
            sts = ServerRepositoryStatus()
            if sts.is_tainted(dbfile) and not sts.is_unlock_msg(dbfile):
                u_msg = "[%s] %s" % (brown(repo),
                    darkgreen(_("mirrors have not been unlocked. Sync them.")),)
                self._server.output(
                    u_msg,
                    importance = 1,
                    level = "warning",
                    header = brown(" * ")
                )
                # avoid spamming
                sts.set_unlock_msg(dbfile)

        return 0

    def commit_hook(self, entropy_repository_instance):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling commit_hook => %s" % (
                self,)
            )

        dbs = ServerRepositoryStatus()
        dbfile = self._metadata['local_dbfile']
        if dbfile is None:
            # fake repo, or temporary one
            return 0
        repo = entropy_repository_instance.repository_id()
        read_only = self._metadata['read_only']
        if read_only:
            # do not taint database
            return 0

        # taint the database status
        taint_file = self._server._get_local_repository_taint_file(repo)
        enc = etpConst['conf_encoding']
        with codecs.open(taint_file, "w", encoding=enc) as f:
            f.write("repository tainted\n")

        const_setup_file(taint_file, etpConst['entropygid'], 0o664)
        dbs.set_tainted(dbfile)

        if not dbs.is_bumped(dbfile):
            # bump revision, setting DatabaseBump causes
            # the session to just bump once
            dbs.set_bumped(dbfile)
            """
            Entropy repository revision bumping function.
            Every time it's called,
            revision is incremented by 1.
            """
            revision_file = self._server._get_local_repository_revision_file(
                repo)
            enc = etpConst['conf_encoding']
            if not os.path.isfile(revision_file):
                revision = 1
            else:
                with codecs.open(revision_file, "r", encoding=enc) as rev_f:
                    revision = int(rev_f.readline().strip())
                    revision += 1

            tmp_revision_file = revision_file + ".tmp"
            with codecs.open(tmp_revision_file, "w", encoding=enc) as rev_fw:
                rev_fw.write(str(revision)+"\n")

            # atomic !
            os.rename(tmp_revision_file, revision_file)

        if not dbs.are_sets_synced(dbfile):
            # auto-update package sets
            self._server._sync_package_sets(entropy_repository_instance)
            dbs.set_synced_sets(dbfile)

        return 0

    def _get_category_description_from_disk(self, category):
        """
        Get category name description from Source Package Manager.

        @param category: category name
        @type category: string
        @return: category description
        @rtype: string
        """
        spm = self._server.Spm()
        return spm.get_package_category_description_metadata(category)

    def __save_rss(self, srv_repo, rss_name, srv_updates):
        # save to disk
        try:
            self._cacher.save(rss_name, srv_updates,
                cache_dir = Server.CACHE_DIR)
        except IOError as err:
            e_msg = "[%s] %s: %s" % (brown(srv_repo),
                purple(_("cannot store updates RSS cache")),
                repr(err),)
            self._server.output(
                e_msg,
                importance = 1,
                level = "warning",
                header = brown(" * ")
            )

    def _write_rss_for_removed_package(self, repo_db, package_id):

        # setup variables we're going to use
        srv_repo = repo_db.repository_id()
        rss_revision = repo_db.retrieveRevision(package_id)
        rss_atom = "%s~%s" % (repo_db.retrieveAtom(package_id), rss_revision,)
        status = ServerRepositoryStatus()
        srv_updates = status.get_updates_log(srv_repo)
        rss_name = srv_repo + etpConst['rss-dump-name']

        # load metadata from on disk cache, if available
        rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR)
        if rss_obj:
            srv_updates.update(rss_obj)

        # setup metadata keys, if not available
        if 'added' not in srv_updates:
            srv_updates['added'] = {}
        if 'removed' not in srv_updates:
            srv_updates['removed'] = {}
        if 'light' not in srv_updates:
            srv_updates['light'] = {}

        # if pkgatom (rss_atom) is in the "added" metadata, drop it
        if rss_atom in srv_updates['added']:
            del srv_updates['added'][rss_atom]
        # same thing for light key
        if rss_atom in srv_updates['light']:
            del srv_updates['light'][rss_atom]

        # add metadata
        mydict = {}
        try:
            mydict['description'] = repo_db.retrieveDescription(package_id)
        except TypeError:
            mydict['description'] = "N/A"
        try:
            mydict['homepage'] = repo_db.retrieveHomepage(package_id)
        except TypeError:
            mydict['homepage'] = ""
        srv_updates['removed'][rss_atom] = mydict

        # save to disk
        self.__save_rss(srv_repo, rss_name, srv_updates)

    def _write_rss_for_added_package(self, repo_db, package_id, package_data):

        # setup variables we're going to use
        srv_repo = repo_db.repository_id()
        rss_atom = "%s~%s" % (package_data['atom'], package_data['revision'],)
        status = ServerRepositoryStatus()
        srv_updates = status.get_updates_log(srv_repo)
        rss_name = srv_repo + etpConst['rss-dump-name']

        # load metadata from on disk cache, if available
        rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR)
        if rss_obj:
            srv_updates.update(rss_obj)

        # setup metadata keys, if not available
        if 'added' not in srv_updates:
            srv_updates['added'] = {}
        if 'removed' not in srv_updates:
            srv_updates['removed'] = {}
        if 'light' not in srv_updates:
            srv_updates['light'] = {}

        # if package_data['atom'] (rss_atom) is in the
        # "removed" metadata, drop it
        if rss_atom in srv_updates['removed']:
            del srv_updates['removed'][rss_atom]

        # add metadata
        srv_updates['added'][rss_atom] = {}
        srv_updates['added'][rss_atom]['description'] = \
            package_data['description']
        srv_updates['added'][rss_atom]['homepage'] = \
            package_data['homepage']

        srv_updates['light'][rss_atom] = {}
        srv_updates['light'][rss_atom]['description'] = \
            package_data['description']
        srv_updates['light'][rss_atom]['homepage'] = \
            package_data['homepage']
        srv_updates['light'][rss_atom]['package_id'] = package_id
        date_raw_str = const_convert_to_rawstring(package_data['datecreation'])
        srv_updates['light'][rss_atom]['time_hash'] = \
            hashlib.sha256(date_raw_str).hexdigest()

        # save to disk
        self.__save_rss(srv_repo, rss_name, srv_updates)

    def add_package_hook(self, entropy_repository_instance, package_id,
        package_data):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling add_package_hook => %s" % (
                self,)
            )

        # handle server-side repo RSS support
        sys_set_plug = self.srv_sys_settings_plugin
        if self._settings[sys_set_plug]['server']['rss']['enabled']:
            self._write_rss_for_added_package(entropy_repository_instance,
                package_id, package_data)

        try:
            descdata = self._get_category_description_from_disk(
                package_data['category'])
            entropy_repository_instance.setCategoryDescription(
                package_data['category'], descdata)
        except (IOError, OSError, EOFError,):
            pass
        entropy_repository_instance.commit()

        return 0

    def remove_package_hook(self, entropy_repository_instance, package_id,
        from_add_package):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling remove_package_hook => %s" % (
                self,)
            )

        # handle server-side repo RSS support
        sys_set_plug = self.srv_sys_settings_plugin
        if self._settings[sys_set_plug]['server']['rss']['enabled'] \
            and (not from_add_package):

            # store addPackage action
            self._write_rss_for_removed_package(entropy_repository_instance,
                package_id)

        return 0

    def treeupdates_move_action_hook(self, entropy_repository_instance,
        package_id):
        # check for injection and warn the developer
        injected = entropy_repository_instance.isInjected(package_id)
        new_atom = entropy_repository_instance.retrieveAtom(package_id)
        if injected:
            mytxt = "%s: %s %s. %s !!! %s." % (
                bold(_("INJECT")),
                blue(str(new_atom)),
                red(_("has been injected")),
                red(_("quickpkg manually to update embedded db")),
                red(_("Repository updated anyway")),
            )
            self._server.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = darkred(" * ")
            )
        return 0

    def treeupdates_slot_move_action_hook(self, entropy_repository_instance,
        package_id):
        return self.treeupdates_move_action_hook(entropy_repository_instance,
            package_id)


class RepositoryConfigParser(BaseConfigParser):
    """
    Entropy .ini-like server-side repository configuration file parser.

    Entropy Server now supports repositories defined inside
    /etc/entropy/repositories.conf.d/ files, written using the
    syntax detailed below. This improves the ability to enable, disable,
    add and remove repositories programmatically. Furthermore, it
    makes possible to extend the supported parameters without breaking
    backward compatibility.

    In order to differentiate Entropy Client repository definitions between
    Entropy Server ones, each repository section must start with "[server=".

    This is an example of the syntax (with a complete listing
    of the supported arguments):

    [server=sabayon-limbo]
    desc = Sabayon Linux Official Testing Repository
    repo = ssh://username@full.host:~username/sabayon-limbo
    enabled = <true/false>

    [server=sabayon-limbo]
    desc = This statement will be ignored.
    repo-only = ssh://username@repo.host:~username/sabayon-limbo
    pkg-only = ssh://username@pkg.host:~username/sabayon-limbo

    [server=sabayon-base]
    desc = This is the base repository.
    repo-only = ssh://username@repo.host:~username/sabayon-base
    pkg-only = ssh://username@pkg.host:~username/sabayon-base
    base = <true/false>

    As you can see, multiple statements for the same repository
    are allowed. However, only the first desc = statement will be
    considered, while there can be as many {pkg,repo}* = as you want.

    The repository order is important, but this is guaranteed by the
    fact that configuration files are parsed in lexical order.

    Statements description:
    - "desc": stands for description, the repository name description.
    - "repo": the push & pull URI, for both packages and repository database.
    - "repo-only": same as repo, but only for the repository database
                   push & pull.
    - "pkg-only": same as repo, but only for the packages push & pull.
             The supported protocols are those supported by entropy.fetchers.
    - "enabled": if set, its value can be either "true" or "false". The default
                 value is "true". It indicates if a repository is configured
                 but currently disabled or enabled. Please take into account
                 that config files in /etc/entropy/repositories.conf.d/ starting
                 with "_" are considered to contain disabled repositories. This
                 is just provided for convienence.
    - "base": if set, its value can be either "true" or "false". The default
              value is "false". If no repository has the flag set, the first
              listed repository will be the base one. Only the first repository
              with "base = true" will be considered. The base repository is the
              repository that is considered base for all the others
              (the main one).
    - "exclude-qa": if set, its value can be either "true" or "false".
                    The default value is "false". If "true", the repository is
                    excluded from QA checks.
    """

    _SUPPORTED_KEYS = ("desc", "repo", "repo-only", "pkg-only",
                       "base", "enabled", "exclude-qa")

    _DEFAULT_ENABLED_VALUE = True
    _DEFAULT_QA_VALUE = False
    _DEFAULT_BASE_VALUE = False

    # Repository configuration file suggested prefix. If config files
    # are prefixed with this string, they can be automatically handled
    # by Entropy.
    FILENAME_PREFIX = "entropysrv_"

    def __init__(self, encoding = None):
        super(RepositoryConfigParser, self).__init__(encoding = encoding)

    @classmethod
    def _validate_section(cls, match):
        """
        Reimpemented from BaseConfigParser.
        """
        # a new repository begins
        groups = match.groups()
        if not groups:
            return

        candidate = groups[0]
        prefix = "server="
        if not candidate.startswith(prefix):
            return
        candidate = candidate[len(prefix):]
        if not entropy.tools.validate_repository_id(candidate):
            return
        return candidate

    def base_repository(self):
        """
        Return the base repository, if any, or None.

        @return: the base repository identifier
        @rtype: string or None
        """
        repositories = self.repositories()
        base = None
        for repository_id in repositories:
            try:
                p_value = self[repository_id]["base"][0]
                value = False
                if p_value.strip().lower() == "true":
                    value = True
            except KeyError:
                value = self._DEFAULT_BASE_VALUE
            if value:
                base = repository_id
                break

        if base is None and repositories:
            base = repositories[0]
        return base

    def add(self, repository_id, desc, repo, repo_only, pkg_only,
            base, enabled = True, exclude_qa = False):
        """
        Add a repository to the repository configuration files directory.
        Older repository configuration may get overwritten. This method
        only writes repository configuration in the new .ini format and to
        /etc/entropy/repositories.conf.d/<filename prefix><repository id>.

        @param repository_id: repository identifier
        @type repository_id: string
        @param desc: repository description
        @type desc: string
        @param repo: list of "repo=" uris
        @type repo: list
        @param repo_only: list of "repo-only=" uris
        @type repo_only: list
        @param pkg_only: list of "pkg-only=" uris
        @type pkg_only: list
        @param base: True, if this is the base repository
        @type base: bool
        @keyword enabled: True, if the repository is enabled
        @type enabled: bool
        @keyword exclude_qa: True, if the repository should be excluded from QA
        @type exclude_qa: bool
        """
        settings = SystemSettings()
        repo_d_conf = settings.get_setting_dirs_data()['repositories_conf_d']
        conf_d_dir, _conf_files_mtime, _skipped_files, _auto_upd = repo_d_conf
        # as per specifications, enabled config files handled by
        # Entropy Server (see repositories.conf.d/README) start with
        # entropysrv_ prefix.
        base_name = self.FILENAME_PREFIX + repository_id
        enabled_conf_file = os.path.join(conf_d_dir, base_name)
        # while disabled config files start with _
        disabled_conf_file = os.path.join(conf_d_dir, "_" + base_name)

        self.write(enabled_conf_file, repository_id, desc, repo, repo_only,
                   pkg_only, base, enabled = enabled, exclude_qa = exclude_qa)

        # if any disabled entry file is around, kill it with fire!
        try:
            os.remove(disabled_conf_file)
        except OSError as err:
            if err.errno != errno.ENOENT:
                raise

        return True

    def remove(self, repository_id):
        """
        Remove a repository from the repositories configuration files directory.

        This method only removes repository configuration at
        /etc/entropy/repositories.conf.d/<filename prefix><repository id>.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: True, if success
        @rtype: bool
        """
        settings = SystemSettings()
        repo_d_conf = settings.get_setting_dirs_data()['repositories_conf_d']
        conf_d_dir, _conf_files_mtime, _skipped_files, _auto_upd = repo_d_conf
        # as per specifications, enabled config files handled by
        # Entropy Server (see repositories.conf.d/README) start with
        # entropysrv_ prefix.
        base_name = self.FILENAME_PREFIX + repository_id
        enabled_conf_file = os.path.join(conf_d_dir, base_name)
        # while disabled config files start with _
        disabled_conf_file = os.path.join(conf_d_dir, "_" + base_name)

        accomplished = False
        try:
            os.remove(enabled_conf_file)
            accomplished = True
        except OSError as err:
            if err.errno != errno.ENOENT:
                raise

        # since we want to remove, also drop disabled
        # config files
        try:
            os.remove(disabled_conf_file)
            accomplished = True
        except OSError as err:
            if err.errno != errno.ENOENT:
                raise

        return accomplished

    def write(self, path, repository_id, desc, repo, repo_only,
              pkg_only, base, enabled = True, exclude_qa = False):
        """
        Write the repository configuration to the given file.

        @param path: configuration file to write
        @type path: string
        @param repository_id: repository identifier
        @type repository_id: string
        @param desc: repository description
        @type desc: string
        @param repo: list of "repo=" uris
        @type repo: list
        @param repo_only: list of "repo-only=" uris
        @type repo_only: list
        @param pkg_only: list of "pkg-only=" uris
        @type pkg_only: list
        @param base: True, if this is the base repository, False if not, None
            if unset.
        @type base: bool
        @keyword enabled: True, if the repository is enabled
        @type enabled: bool
        @keyword exclude_qa: True, if the repository should be excluded from QA
        @type exclude_qa: bool
        """
        if enabled:
            enabled_str = "true"
        else:
            enabled_str = "false"

        if exclude_qa:
            qa_str = "true"
        else:
            qa_str = "false"

        if base:
            base_str = "base = true"
        elif base is None:
            base_str = "# base = false"
        else:
            base_str = "base = false"

        repos_str = ""
        for r in repo:
            repos_str += "repo = %s\n" % (r,)

        repo_only_str = ""
        for r in repo_only:
            repo_only_str += "repo-only = %s\n" % (r,)
        if not repo_only_str:
            repo_only_str = "# repo-only = "

        pkg_only_str = ""
        for pkg in pkg_only:
            pkg_only_str += "pkg-only = %s\n" % (pkg,)
        if not pkg_only_str:
            pkg_only_str = "# pkg-only = "

        meta = {
            "repository_id": repository_id,
            "desc": desc,
            "repos": repos_str.rstrip(),
            "repo_only": repo_only_str.rstrip(),
            "pkg_only": pkg_only_str.rstrip(),
            "enabled": enabled_str,
            "exclude_qa": qa_str,
            "base": base_str,
        }

        config = """\
# Repository configuration file automatically generated
# by Entropy Server on your behalf.

[server=%(repository_id)s]
%(base)s
exclude-qa = %(exclude_qa)s
desc = %(desc)s
%(repos)s
%(repo_only)s
%(pkg_only)s
enabled = %(enabled)s
""" % meta

        entropy.tools.atomic_write(path, config, self._encoding)

    def repositories(self):
        """
        Return a list of valid parsed repositories.

        A repository is considered valid iff it contains
        at least "repo". The parse order is preserved.
        """
        required_keys = set(("repo",))
        repositories = []

        for repository_id in self._ordered_sections:
            repo_data = self[repository_id]
            remaining = required_keys - set(repo_data.keys())
            if not remaining:
                # then required_keys are there
                repositories.append(repository_id)

        return repositories

    def repo(self, repository_id):
        """
        Return the repository push & pull URIs for both packages and
        repository database.

        @param repository_id: the repository identifier
        @type repository_id: string
        @raise KeyError: if repository_id is not found or
            metadata is not available
        @return: the repository push & pull URIs.
        @rtype: list
        """
        return self[repository_id]["repo"]

    def repo_only(self, repository_id):
        """
        Return the repository push & pull URIs for the repository
        database only.

        @param repository_id: the repository identifier
        @type repository_id: string
        @raise KeyError: if repository_id is not found or
            metadata is not available
        @return: the repository push & pull URIs for the repository
            database only.
        @rtype: list
        """
        return self[repository_id]["repo-only"]

    def pkg_only(self, repository_id):
        """
        Return the repository push & pull URIs for the repository only.

        @param repository_id: the repository identifier
        @type repository_id: string
        @raise KeyError: if repository_id is not found or
            metadata is not available
        @return: the repository push & pull URIs for the packages only.
        @rtype: list
        """
        return self[repository_id]["pkg-only"]

    def desc(self, repository_id):
        """
        Return the description of the repository.

        @param repository_id: the repository identifier
        @type repository_id: string
        @raise KeyError: if repository_id is not found or
            metadata is not available
        @return: the repository description
        @rtype: string
        """
        return self[repository_id]["desc"][0]

    def enabled(self, repository_id):
        """
        Return whether the repository is enabled or disabled.

        @param repository_id: the repository identifier
        @type repository_id: string
        @return: the repository status
        @rtype: bool
        """
        try:
            enabled = self[repository_id]["enabled"][0]
            return enabled.strip().lower() == "true"
        except KeyError:
            return self._DEFAULT_ENABLED_VALUE

    def exclude_qa(self, repository_id):
        """
        Return whether the repository is excluded from QA.

        @param repository_id: the repository identifier
        @type repository_id: string
        @return: the repository QA exclusion status
        @rtype: bool
        """
        try:
            exclude = self[repository_id]["exclude-qa"][0]
            return exclude.strip().lower() == "true"
        except KeyError:
            return self._DEFAULT_QA_VALUE


class ServerSystemSettingsPlugin(SystemSettingsPlugin):

    # List of static server-side repositories that must survive
    # a repositories metadata reload
    REPOSITORIES = {}

    def __init__(self, plugin_id, helper_interface):
        SystemSettingsPlugin.__init__(self, plugin_id, helper_interface)
        self._mtime_cache = {}

    @staticmethod
    def server_conf_path():
        """
        Return current server.conf path, this takes into account the current
        configuration files directory path (which is affected by "root" path
        changes [default: /])
        """
        # path to /etc/entropy/server.conf (usually, depends on systemroot)
        return os.path.join(etpConst['confdir'], "server.conf")

    @classmethod
    def analyze_server_repo_string(cls, repostring, product = None):
        """
        Analyze a server repository string (usually contained in server.conf),
        extracting all the parameters.

        @param repostring: repository string
        @type repostring: string
        @keyword product: system product which repository belongs to
        @rtype: None
        @return: None
        """

        if product is None:
            product = etpConst['product']

        data = {}
        repo_key, repostring = entropy.tools.extract_setting(repostring)
        if repo_key != "repository":
            raise AttributeError("invalid repostring passed")

        repo_split = repostring.split("|")
        if len(repo_split) < 3:
            raise AttributeError("invalid repostring passed (2)")

        repository_id = repo_split[0].strip()
        desc = repo_split[1].strip()
        uris = repo_split[2].strip().split()
        exclude_qa = False  # not supported through server.conf

        repo_mirrors = []
        pkg_mirrors = []
        for uri in uris:
            do_pkg = False
            do_repo = False
            while True:
                if uri.startswith("<p>"):
                    do_pkg = True
                    uri = uri[3:]
                    continue
                if uri.startswith("<r>"):
                    do_repo = True
                    uri = uri[3:]
                    continue
                break

            if not (do_repo or do_pkg):
                do_repo = True
                do_pkg = True
            if do_repo:
                repo_mirrors.append(uri)
            if do_pkg:
                pkg_mirrors.append(uri)

        return repository_id, cls._generate_repository_metadata(
            repository_id, desc, repo_mirrors, pkg_mirrors, exclude_qa)

    @classmethod
    def _generate_repository_metadata(cls, repository_id, desc,
                                      repo_mirrors, pkg_mirrors,
                                      exclude_qa):
        """
        Generate the repository metadata given raw information.

        @param repository_id: the repository identifier
        @type repository_id: string
        @param desc: repository description
        @type desc: string
        @param repo_mirrors: list of repository database mirrors
        @type repo_mirrors: list
        @param pkg_mirrors: list of repository packages mirrors
        @type pkg_mirrors: list
        @param exclude_qa: exclude from QA checks
        @type exclude_qa: bool
        @return: the repository metadata
        @rtype: dict
        """
        data = {}
        data['repoid'] = repository_id
        data['description'] = desc
        data['pkg_mirrors'] = pkg_mirrors[:]
        data['repo_mirrors'] = repo_mirrors[:]
        data['community'] = False
        data['exclude_qa'] = exclude_qa
        return data

    def __generic_parser(self, filepath):
        """
        Internal method. This is the generic file parser here.

        @param filepath: valid path
        @type filepath: string
        @return: raw text extracted from file
        @rtype: list
        """
        root = etpConst['systemroot']
        try:
            mtime = os.path.getmtime(filepath)
        except (OSError, IOError):
            mtime = 0.0

        cache_key = (root, filepath)
        cache_obj = self._mtime_cache.get(cache_key)
        if cache_obj is not None:
            if cache_obj['mtime'] == mtime:
                return cache_obj['data']

        cache_obj = {'mtime': mtime,}

        enc = etpConst['conf_encoding']
        data = entropy.tools.generic_file_content_parser(filepath,
            comment_tag = "##", encoding = enc)
        if SystemSettings.DISK_DATA_CACHE:
            cache_obj['data'] = data
            self._mtime_cache[cache_key] = cache_obj
        return data

    def get_updatable_configuration_files(self, repository_id):
        """
        Overridden from SystemSettings.
        """
        files = set()
        # hope that all the repos get synchronized with respect to
        # package names moves
        dep_rewrite_file = Server._get_conf_dep_rewrite_file()
        dep_blacklist_file = Server._get_conf_dep_blacklist_file()
        files.add(dep_rewrite_file)
        files.add(dep_blacklist_file)

        if (repository_id is not None) and \
            (repository_id in self._helper.repositories()):

            critical_file = self._helper._get_local_critical_updates_file(
                repository_id)
            files.add(critical_file)
            keywords_file = self._helper._get_local_repository_keywords_file(
                repository_id)
            files.add(keywords_file)
            mask_file = self._helper._get_local_repository_mask_file(
                repository_id)
            files.add(mask_file)
            bl_file = self._helper._get_missing_dependencies_blacklist_file(
                repository_id)
            files.add(bl_file)
            restricted_file = self._helper._get_local_restricted_file(
                repository_id)
            files.add(restricted_file)
            system_mask_file = \
                self._helper._get_local_repository_system_mask_file(
                    repository_id)
            files.add(system_mask_file)

        return files

    def dep_rewrite_parser(self, sys_set):

        cached = getattr(self, '_mod_rewrite_data', None)
        if cached is not None:
            return cached

        data = {}
        rewrite_file = Server._get_conf_dep_rewrite_file()
        if not os.path.isfile(rewrite_file):
            return data
        rewrite_content = self.__generic_parser(rewrite_file)

        for line in rewrite_content:
            params = line.split()
            if len(params) < 2:
                continue
            pkg_match, pattern, replaces = params[0], params[1], params[2:]
            if pattern.startswith("++"):
                compiled_pattern = None
                pattern = pattern[2:]
                if not pattern:
                    # malformed
                    continue
            else:
                try:
                    compiled_pattern = re.compile(pattern)
                except re.error:
                    # invalid pattern
                    continue
            # use this key to make sure to not overwrite similar entries
            data[(pkg_match, pattern)] = (compiled_pattern, replaces)

        self._mod_rewrite_data = data
        return data

    def dep_blacklist_parser(self, sys_set):

        data = {}
        blacklist_file = Server._get_conf_dep_blacklist_file()
        if not os.path.isfile(blacklist_file):
            return data
        blacklist_content = self.__generic_parser(blacklist_file)

        for line in blacklist_content:
            params = line.split()
            if len(params) < 2:
                continue
            pkg_match, blacklisted_deps = params[0], params[1:]
            # use this key to make sure to not overwrite similar entries
            obj = data.setdefault(pkg_match, [])
            obj.extend(blacklisted_deps)

        return data

    def qa_sets_parser(self, sys_set):

        data = {}
        sets_file = Server._get_conf_qa_sets_file()
        if not os.path.isfile(sets_file):
            return data
        sets_content = self.__generic_parser(sets_file)

        for line in sets_content:
            params = line.split()
            if len(params) < 2:
                continue
            repo_id, qa_sets = params[0], params[1:]
            obj = data.setdefault(repo_id, set())
            obj.update(qa_sets)

        return data

    def server_parser(self, sys_set):
        """
        Parses Entropy server system configuration file.

        @return dict data
        """
        srv_plugin_class = ServerSystemSettingsPlugin
        server_conf = srv_plugin_class.server_conf_path()
        root = etpConst['systemroot']
        try:
            mtime = os.path.getmtime(server_conf)
        except (OSError, IOError):
            mtime = 0.0

        enc = etpConst['conf_encoding']
        serverconf = None
        cache_key = (root, server_conf)
        cache_obj = self._mtime_cache.get(cache_key)
        if cache_obj is not None:
            if cache_obj['mtime'] == mtime:
                serverconf = cache_obj['data']
        else:
            cache_obj = {'mtime': mtime,}

        if serverconf is None:
            try:
                with codecs.open(server_conf, "r", encoding=enc) \
                        as server_f:
                    serverconf = [x.strip() for x in server_f.readlines() \
                                      if x.strip()]
            except IOError as err:
                if err.errno != errno.ENOENT:
                    raise
                # if file doesn't exist, provide empty
                # serverconf list. In this way, we make sure that
                # any additional metadata gets added.
                # see the for loop iterating through the
                # repository identifiers
                serverconf = []

            if SystemSettings.DISK_DATA_CACHE:
                cache_obj['data'] = serverconf
                self._mtime_cache[cache_key] = cache_obj

        data = {
            'repositories': srv_plugin_class.REPOSITORIES.copy(),
            'community_mode': False,
            'qa_langs': [const_convert_to_unicode("en_US"),
                         const_convert_to_unicode("C")],
            'default_repository_id': const_convert_to_unicode(
                etpConst['defaultserverrepositoryid']),
            'base_repository_id': None,
            'packages_expiration_days': etpConst['packagesexpirationdays'],
            'database_file_format': const_convert_to_unicode(
                etpConst['etpdatabasefileformat']),
            'disabled_eapis': set(),
            'broken_revdeps_qa_check': True,
            'exp_based_scope': etpConst['expiration_based_scope'],
            # disabled by default for now
            'nonfree_packages_dir_support': False,
            'sync_speed_limit': None,
            'weak_package_files': False,
            'changelog': True,
            'rss': {
                'enabled': etpConst['rss-feed'],
                'name': const_convert_to_unicode(etpConst['rss-name']),
                'light_name': const_convert_to_unicode(
                    etpConst['rss-light-name']),
                'base_url': const_convert_to_unicode(etpConst['rss-base-url']),
                'website_url': const_convert_to_unicode(
                    etpConst['rss-website-url']),
                'editor': const_convert_to_unicode(
                    etpConst['rss-managing-editor']),
                'max_entries': etpConst['rss-max-entries'],
                'light_max_entries': etpConst['rss-light-max-entries'],
            },
        }

        fake_instance = self._helper.fake_default_repo
        default_repo_changed = False

        def _offservrepoid(line, setting):
            # NOTE: remove this in future, supported for backward compat.
            # NOTE: added for backward and mixed compat.
            if default_repo_changed:
                return
            if not fake_instance:
                data['default_repository_id'] = setting.strip()

        def _default_repo(line, setting):
            if not fake_instance:
                data['default_repository_id'] = setting.strip()
            default_repo_changed = True

        def _exp_days(line, setting):
            mydays = setting.strip()
            try:
                mydays = int(mydays)
                data['packages_expiration_days'] = mydays
            except ValueError:
                return

        def _exp_based_scope(line, setting):
            exp_opt = entropy.tools.setting_to_bool(setting)
            if exp_opt is not None:
                data['exp_based_scope'] = exp_opt

        def _nf_packages_dir_sup(line, setting):
            opt = entropy.tools.setting_to_bool(setting)
            if opt is not None:
                data['nonfree_packages_dir_support'] = opt

        def _disabled_eapis(line, setting):
            mydis = setting.strip().split(",")
            try:
                mydis = [int(x) for x in mydis]
                mydis = set([x for x in mydis if x in (1, 2, 3,)])
            except ValueError:
                return
            if (len(mydis) < 3) and mydis:
                data['disabled_eapis'] = mydis

        def _server_basic_lang(line, setting):
            data['qa_langs'] = setting.strip().split()

        def _broken_revdeps_qa(line, setting):
            opt = entropy.tools.setting_to_bool(setting)
            if opt is not None:
                data['broken_revdeps_qa_check'] = opt

        def _repository_func(line, setting):
            # TODO: deprecate in 2015. repositories.conf.d/ is the
            # supported way to define repositories.
            try:
                repoid, repodata = \
                    srv_plugin_class.analyze_server_repo_string(
                        line, product = sys_set['repositories']['product'])
            except AttributeError:
                # error parsing string
                return

            # validate repository id string
            if not entropy.tools.validate_repository_id(repoid):
                sys.stderr.write("!!! invalid repository id '%s' in '%s'\n" % (
                    repoid, srv_plugin_class.server_conf_path()))
                return

            if repoid in data['repositories']:
                # just update mirrors
                data['repositories'][repoid]['pkg_mirrors'].extend(
                    repodata['pkg_mirrors'])
                data['repositories'][repoid]['repo_mirrors'].extend(
                    repodata['repo_mirrors'])
            else:
                data['repositories'][repoid] = repodata.copy()

            # base_repository_id support
            if data['base_repository_id'] is None:
                data['base_repository_id'] = repoid

        def _database_format(line, setting):
            if setting in etpConst['etpdatabasesupportedcformats']:
                data['database_file_format'] = setting

        def _syncspeedlimit(line, setting):
            try:
                speed_limit = int(setting)
            except ValueError:
                speed_limit = None
            data['sync_speed_limit'] = speed_limit

        def _weak_package_files(line, setting):
            opt = entropy.tools.setting_to_bool(setting)
            if opt is not None:
                data['weak_package_files'] = opt

        def _changelog(line, setting):
            bool_setting = entropy.tools.setting_to_bool(setting)
            if bool_setting is not None:
                data['changelog'] = bool_setting

        def _community_mode(line, setting):
            bool_setting = entropy.tools.setting_to_bool(setting)
            if bool_setting is not None:
                data['community_mode'] = bool_setting

        def _rss_feed(line, setting):
            bool_setting = entropy.tools.setting_to_bool(setting)
            if bool_setting is not None:
                data['rss']['enabled'] = bool_setting

        def _rss_name(line, setting):
            data['rss']['name'] = setting

        def _rss_light_name(line, setting):
            data['rss']['light_name'] = setting

        def _rss_base_url(line, setting):
            data['rss']['base_url'] = setting

        def _rss_website_url(line, setting):
            data['rss']['website_url'] = setting

        def _managing_editor(line, setting):
            data['rss']['editor'] = setting

        def _max_rss_entries(line, setting):
            try:
                entries = int(setting)
                data['rss']['max_entries'] = entries
            except (ValueError, IndexError,):
                return

        def _max_rss_light_entries(line, setting):
            try:
                entries = int(setting)
                data['rss']['light_max_entries'] = entries
            except (ValueError, IndexError,):
                return

        settings_map = {
            'officialserverrepositoryid': _offservrepoid,
            'default-repository': _default_repo,
            'expiration-days': _exp_days,
            'community-mode': _community_mode,
            'expiration-based-scope': _exp_based_scope,
            'nonfree-packages-directory-support': _nf_packages_dir_sup,
            'disabled-eapis': _disabled_eapis,
            'broken-reverse-deps': _broken_revdeps_qa,
            'server-basic-languages': _server_basic_lang,
            'repository': _repository_func,
            'database-format': _database_format,
            # backward compatibility
            'sync-speed-limit': _syncspeedlimit,
            'syncspeedlimit': _syncspeedlimit,
            'weak-package-files': _weak_package_files,
            'changelog': _changelog,
            'rss-feed': _rss_feed,
            'rss-name': _rss_name,
            'rss-light-name': _rss_light_name,
            'rss-base-url': _rss_base_url,
            'rss-website-url': _rss_website_url,
            'managing-editor': _managing_editor,
            'max-rss-entries': _max_rss_entries,
            'max-rss-light-entries': _max_rss_light_entries,
        }

        for line in serverconf:

            key, value = entropy.tools.extract_setting(line)
            if key is None:
                continue

            func = settings_map.get(key)
            if func is None:
                continue
            func(line, value)

        # .ini-like file support.
        repositories_d_conf = sys_set.get_setting_dirs_data(
            )['repositories_conf_d']
        _conf_dir, setting_files, _skipped_files, _upd = repositories_d_conf
        candidate_inis = [x for x,y in setting_files]

        ini_parser = RepositoryConfigParser(encoding = enc)
        try:
            ini_parser.read(candidate_inis)
        except (IOError, OSError) as err:
            sys.stderr.write("Cannot parse %s: %s\n" % (
                    " ".join(candidate_inis),
                    err))
            ini_parser = None

        if ini_parser:
            repositories = set(data['repositories'].keys())
            ini_repositories = ini_parser.repositories()
            if data['base_repository_id'] is None:
                # if base_repository_id is not set, then
                # take the value of ini config files.
                ini_base = ini_parser.base_repository()
                if ini_base:
                    data['base_repository_id'] = ini_base

            for ini_repository in ini_repositories:
                if ini_repository in repositories:
                    # double syntax is not supported.
                    continue
                ini_enabled = ini_parser.enabled(ini_repository)
                if not ini_enabled:
                    continue

                ini_exclude_qa = ini_parser.exclude_qa(ini_repository)

                try:
                    ini_desc = ini_parser.desc(ini_repository)
                except KeyError:
                    ini_desc = _("No description")
                try:
                    ini_mirrors = ini_parser.repo(ini_repository)
                except KeyError:
                    ini_mirrors = []

                repo_mirrors = []
                pkg_mirrors = []
                repo_mirrors.extend(ini_mirrors)
                pkg_mirrors.extend(ini_mirrors)

                try:
                    repo_mirrors.extend(ini_parser.repo_only(ini_repository))
                except KeyError:
                    pass
                try:
                    pkg_mirrors.extend(ini_parser.pkg_only(ini_repository))
                except KeyError:
                    pass

                repo_data = srv_plugin_class._generate_repository_metadata(
                    ini_repository, ini_desc, repo_mirrors, pkg_mirrors,
                    ini_exclude_qa)
                data['repositories'][ini_repository] = repo_data

        env_community_mode = os.getenv("ETP_COMMUNITY_MODE")
        if env_community_mode == "0":
            data['community_mode'] = False
        elif env_community_mode == "1":
            data['community_mode'] = True

        # add system database if community repository mode is enabled
        if data['community_mode']:
            client_repository_id = InstalledPackagesRepository.NAME

            mydata = srv_plugin_class._generate_repository_metadata(
                client_repository_id,
                const_convert_to_unicode(
                    "Community Repositories System Repository"),
                [],[], False)

            data['repositories'][client_repository_id] = mydata
            srv_plugin_class.REPOSITORIES[client_repository_id] = \
                mydata
            # installed packages repository is now the base repository
            data['base_repository_id'] = client_repository_id

        # expand paths
        for repoid in data['repositories']:
            srv_plugin_class.extend_repository_metadata(
                sys_set, repoid, data['repositories'][repoid])

        # Support for shell variables
        shell_repoid = os.getenv('ETP_REPO')
        if shell_repoid:
            data['default_repository_id'] = shell_repoid

        expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
        if expiration_days:
            try:
                expiration_days = int(expiration_days)
                data['packages_expiration_days'] = expiration_days
            except ValueError:
                pass

        return data

    @staticmethod
    def extend_repository_metadata(system_settings, repository_id, metadata):
        """
        Extend server-side Repository metadata dictionary
        with information required by Entropy Server.
        """
        metadata['repo_basedir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id)

        metadata['packages_dir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            etpConst['packagesrelativepath_basedir'],
            etpConst['currentarch'])

        metadata['packages_dir_nonfree'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            etpConst['packagesrelativepath_basedir_nonfree'],
            etpConst['currentarch'])

        metadata['packages_dir_restricted'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            etpConst['packagesrelativepath_basedir_restricted'],
            etpConst['currentarch'])

        metadata['store_dir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            "store",
            etpConst['currentarch'])

        # consider this a base dir
        metadata['upload_basedir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            "upload")

        metadata['database_dir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            "database",
            etpConst['currentarch'])

        metadata['remote_repo_basedir'] = os.path.join(
            system_settings['repositories']['product'],
            repository_id)

        metadata['database_remote_path'] = \
            ServerSystemSettingsPlugin.get_repository_remote_path(
                system_settings, repository_id)
        metadata['override_database_remote_path'] = None

    @staticmethod
    def get_repository_remote_path(system_settings, repository_id):
        return system_settings['repositories']['product'] + "/" + \
            repository_id + "/database/" + etpConst['currentarch']

    @staticmethod
    def set_override_remote_repository(system_settings, repository_id,
        override_repository_id):
        """
        Used to set an overridden remote path where to push repository
        database. This can be used for quickly testing repository changes
        without directly overwriting the real repository.
        """
        repo_path = ServerSystemSettingsPlugin.get_repository_remote_path(
            system_settings, override_repository_id)

        sys_settings_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin']
        srv_data = system_settings[sys_settings_plugin_id]['server']
        repo_data = srv_data['repositories'][repository_id]
        repo_data['override_database_remote_path'] = repo_path


class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):

    def repos_parser(self, sys_set):

        cached = getattr(self, '_repos_data', None)
        if cached is not None:
            return cached

        data = {}
        srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
        # if support is not enabled, don't waste time scanning files
        srv_parser_data = sys_set[srv_plug_id]['server']
        if not srv_parser_data['exp_based_scope']:
            return data

        # get expiration-based packages removal data from config files
        for repoid in srv_parser_data['repositories']:

            # filter out system repository if community repository
            # mode is enabled
            if repoid == InstalledPackagesRepository.NAME:
                continue

            package_ids = set()
            exp_fp = self._helper._get_local_exp_based_pkgs_rm_whitelist_file(
                repoid)
            try:
                dbconn = self._helper.open_server_repository(
                    repoid, just_reading = True)
            except RepositoryError:
                # ignore
                continue

            pkgs = []
            if const_file_readable(exp_fp):
                # don't worry about the race.
                pkgs += entropy.tools.generic_file_content_parser(
                    exp_fp, encoding = etpConst['conf_encoding'])
            if '*' in pkgs: # wildcard support
                package_ids.add(-1)
            else:
                for pkg in pkgs:
                    package_id, rc_match = dbconn.atomMatch(pkg)
                    if rc_match:
                        continue
                    package_ids.add(package_id)

            data[repoid] = package_ids

        self._repos_data = data
        return data

class ServerFakeClientSystemSettingsPlugin(SystemSettingsPlugin):

    def fake_cli_parser(self, sys_set):
        """
        This is just fake, doesn't bring any new metadata but just tweak
        Entropy client ones.
        """
        data = {}
        srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
        # if support is not enabled, don't waste time scanning files
        srv_parser_data = sys_set[srv_plug_id]['server']

        # now setup fake Entropy Client repositories, so that Entropy Server
        # can use Entropy Client interfaces transparently
        srv_repodata = srv_parser_data['repositories']
        cli_repodata = sys_set['repositories']
        # remove unavailable server repos in client metadata first
        cli_repodata['available'].clear()

        for repoid, repo_data in srv_repodata.items():

            try:
                # we must skip the repository validation because
                # repositories have been already validated.
                # moreover, the system repository_id might not be
                # valid (__system__). But still, this is the wanted
                # behaviour.
                xxx, my_data = sys_set._analyze_client_repo_string(
                    "repository = %s|%s|http://--fake--|http://--fake--" \
                        % (repoid, repo_data['description'],),
                    _skip_repository_validation=True)
            except AttributeError as err:
                # yeah, at least let stderr know.
                sys.stderr.write(repr(err) + "\n")
                continue # sorry!

            my_data['repoid'] = repoid
            if '__temporary__' in repo_data:
                # fake repositories, temp ones
                # can't go into Entropy Client, they miss
                # 'database_dir' and other metadata
                my_data['dbpath'] = None
                my_data['__temporary__'] = repo_data['__temporary__']
                my_data['dbrevision'] = 0
            else:
                my_data['dbpath'] = self._helper._get_local_repository_dir(
                    repoid)
                my_data['dbrevision'] = \
                    self._helper.local_repository_revision(
                        repoid)
            cli_repodata['available'][repoid] = my_data

        cli_repodata['default_repository'] = \
            srv_parser_data['default_repository_id']

        del cli_repodata['order'][:]
        if srv_parser_data['base_repository_id'] is not None:
            cli_repodata['order'].append(srv_parser_data['base_repository_id'])
        for repoid in sorted(srv_repodata):
            if repoid not in cli_repodata['order']:
                cli_repodata['order'].append(repoid)

        return data

class ServerQAInterfacePlugin(QAInterfacePlugin):

    def __init__(self, entropy_server_instance):
        self._server = entropy_server_instance

    def __check_package_using_spm(self, package_path):

        spm_class = get_spm_class()
        spm_rc, spm_msg = spm_class.execute_qa_tests(package_path)

        if spm_rc == 0:
            return True
        sys.stderr.write("QA Error: " + spm_msg + "\n")
        sys.stderr.flush()
        return False

    def __extract_edb_analyze_metadata(self, package_path):

        def _is_supported(keywords):
            for arch in etpConst['keywords']:
                if arch in keywords:
                    return True
            return False

        tmp_fd, tmp_f = const_mkstemp(prefix = 'entropy.server')
        dbc = None
        try:
            found_edb = entropy.tools.dump_entropy_metadata(package_path, tmp_f)
            if not found_edb:
                return False
            dbc = self._server._open_temp_repository("test", temp_file = tmp_f,
                initialize = False)
            for package_id in dbc.listAllPackageIds():
                # NOTE: content is tested in entropy.qa builtin package test
                # test content safety
                dbc.retrieveContentSafety(package_id)
                # test keywords
                keywords = dbc.retrieveKeywords(package_id)
                if not _is_supported(keywords):
                    atom = dbc.retrieveAtom(package_id)
                    # big PHAT warning !!
                    self._server.output(darkred("~"*40), level = "warning")
                    self._server.output("[%s, %s] %s" % (
                         brown(os.path.basename(package_path)), teal(atom),
                         purple(_("package has no keyword set, it will be masked !"))),
                        level = "warning", header = darkred(" !!! "))
                    self._server.output(darkred("~"*40), level = "warning")
                    time.sleep(10)
        finally:
            if dbc is not None:
                dbc.close()
            os.close(tmp_fd)

        return True

    def get_tests(self):
        return [self.__check_package_using_spm,
            self.__extract_edb_analyze_metadata]

    def get_id(self):
        return SERVER_QA_PLUGIN


class ServerConfigurationFiles(ConfigurationFiles):

    """
    Subclass Entropy Client version in order to return
    our repository identifiers
    """

    @property
    def _repository_ids(self):
        """
        Return a the list of repository identifiers the object
        is using.
        """
        return self._entropy.repositories()


class Server(Client):

    # Entropy Server cache directory, mainly used for storing commit changes
    CACHE_DIR = os.path.join(etpConst['entropyworkdir'], "server_cache")

    # SystemSettings class variables
    SYSTEM_SETTINGS_PLG_ID = etpConst['system_settings_plugins_ids']['server_plugin']

    # Make possible to disable tree updates completely.
    _inhibit_treeupdates = False

    def init_singleton(self, default_repository = None, save_repository = False,
            fake_default_repo = False, fake_default_repo_id = None,
            fake_default_repo_desc = None, handle_uninitialized = True,
            **kwargs):

        self._indexing = False

        # initialize Entropy Client superclass
        if "installed_repo" not in kwargs:
            kwargs["installed_repo"] = False
        if "repo_validation" not in kwargs:
            kwargs["repo_validation"] = False
        Client.init_singleton(self,
            indexing = self._indexing,
            **kwargs
        )

        if fake_default_repo_desc is None:
            fake_default_repo_desc = 'this is a fake repository'
        self.__instance_destroyed = False

        # settings
        self._memory_db_srv_instances = {}
        self._treeupdates_repos = set()
        self._server_dbcache = {}
        etpSys['serverside'] = True
        self.fake_default_repo = fake_default_repo
        self.fake_default_repo_id = fake_default_repo_id
        self.Mirrors = None
        self._settings_to_backup = []
        self._save_repository = save_repository
        self._sync_lock_cache = set()

        self.sys_settings_fake_cli_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin_fake_client']
        self.sys_settings_fatscope_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin_fatscope']

        # create our SystemSettings plugin
        with self._settings:
            self.sys_settings_plugin = ServerSystemSettingsPlugin(
                Server.SYSTEM_SETTINGS_PLG_ID, self)
            self._settings.add_plugin(self.sys_settings_plugin)

            # Fatscope support SystemSettings plugin
            self.sys_settings_fatscope_plugin = \
                ServerFatscopeSystemSettingsPlugin(
                    self.sys_settings_fatscope_plugin_id, self)
            self._settings.add_plugin(self.sys_settings_fatscope_plugin)

            # Fatscope support SystemSettings plugin
            self.sys_settings_fake_cli_plugin = \
                ServerFakeClientSystemSettingsPlugin(
                    self.sys_settings_fake_cli_plugin_id, self)
            self._settings.add_plugin(self.sys_settings_fake_cli_plugin)

        # setup fake repository
        if fake_default_repo:
            default_repository = fake_default_repo_id
            self._init_generic_memory_server_repository(
                fake_default_repo_id,
                fake_default_repo_desc, set_as_default = True)

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        self._repository = default_repository
        if self._repository is None:
            self._repository = srv_set['default_repository_id']

        if not fake_default_repo:
            if self._repository in srv_set['repositories']:
                try:
                    self._ensure_paths(self._repository)
                except OSError as err:
                    if err.errno != errno.EACCES:
                        raise
                    # sigh, ignore during init

        # if repository is still None, fallback to internal
        # fake repository. This way Entropy Server will work
        # out of the box without any server.conf tweak
        # (and eit bashcomp is happy)
        if self._repository is None:
            repository_id = "__builtin__"
            self._init_generic_memory_server_repository(
                repository_id, "Built-in fallback fake repository",
                set_as_default=True)
            self._repository = repository_id

        if self._repository not in srv_set['repositories']:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                        self._repository,
                        _("repository not configured"),
                    )
            )
        if InstalledPackagesRepository.NAME == self._repository:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                    InstalledPackagesRepository.NAME,
                    _("protected repository id, can't use this, sorry dude..."),
                )
            )

        self.switch_default_repository(
            self._repository, handle_uninitialized=handle_uninitialized)

    def destroy(self, _from_shutdown = False):
        """
        Destroy this singleton instance.
        """
        self.__instance_destroyed = True
        Client.close_repositories(self, mask_clear = False)
        Client.destroy(self, _from_shutdown = _from_shutdown)

        if not _from_shutdown:
            plug_id2 = self.sys_settings_fake_cli_plugin_id
            plug_id1 = self.sys_settings_fatscope_plugin_id
            plug_id = Server.SYSTEM_SETTINGS_PLG_ID
            # reverse insert order
            plugs = [plug_id2, plug_id1, plug_id]
            for plug in plugs:
                if plug is None:
                    continue
                if not self._settings.has_plugin(plug):
                    continue
                self._settings.remove_plugin(plug)

        self.close_repositories()

    @property
    def _cacher(self):
        """
        Return an EntropyCacher object instance.
        """
        return EntropyCacher()

    def is_destroyed(self):
        """
        Return whether the singleton instance is destroyed.
        """
        return self.__instance_destroyed

    def _cache_prefix(self, caller):
        """
        Generate a cache object key prefix to use with EntropyCacher.

        @param caller: a custom function caller name
        @type caller: string
        @return: the cache prefix
        @rtype: string
        """
        return "%s/%s/%s" % (
            __name__, self.__class__.__name__, caller)

    def _get_branch_from_download_relative_uri(self, db_download_uri):
        return db_download_uri.split("/")[2]

    def _swap_branch_in_download_relative_uri(self, new_branch,
        db_download_uri):
        cur_branch = self._get_branch_from_download_relative_uri(
            db_download_uri)
        return db_download_uri.replace("/%s/" % (cur_branch,),
            "/%s/" % (new_branch,))

    def _get_basedir_pkg_listing(self, base_dir, extension, branch = None):

        pkgs_dir_types = set(self._get_pkg_dir_names())
        basedir_raw_content = []
        entropy.tools.recursive_directory_relative_listing(
            basedir_raw_content, base_dir)

        pkg_ext = extension
        pkg_list = [x for x in basedir_raw_content if x.endswith(pkg_ext)]
        pkg_list = [x for x in pkg_list if \
            x.split(os.path.sep)[0] in pkgs_dir_types]

        if branch is not None:
            branch_extractor = \
                self._get_branch_from_download_relative_uri
            pkg_list = [x for x in pkg_list if branch_extractor(x) == branch]

        return pkg_list

    def _get_pkg_dir_names(self):
        return [etpConst['packagesrelativepath_basedir'],
            etpConst['packagesrelativepath_basedir_nonfree'],
            etpConst['packagesrelativepath_basedir_restricted']]

    def _get_remote_repository_relative_path(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['database_remote_path']

    def _get_override_remote_repository_relative_path(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        repo_data = srv_set['repositories'][repository_id]
        return repo_data['override_database_remote_path']

    def _get_local_repository_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasefile'])

    def _get_local_store_directory(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['store_dir']

    def _get_local_upload_directory(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['upload_basedir']

    def _get_local_repository_base_directory(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['repo_basedir']

    def _get_local_repository_taint_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasetaintfile'])

    def _get_local_repository_revision_file(self, repository_id, branch = None):
        return os.path.join(
            self._get_local_repository_dir(repository_id, branch = branch),
                etpConst['etpdatabaserevisionfile'])

    def _get_local_repository_timestamp_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasetimestampfile'])

    def _get_local_repository_mask_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemaskfile'])

    def _get_local_repository_system_mask_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasesytemmaskfile'])

    def _get_local_repository_licensewhitelist_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaselicwhitelistfile'])

    def _get_local_repository_mirrors_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemirrorsfile'])

    def _get_local_repository_fallback_mirrors_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasefallbackmirrorsfile'])

    def _get_local_repository_rss_file(self, repository_id, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), srv_set['rss']['name'])

    def _get_local_repository_changelog_file(self, repository_id,
        branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['changelog_filename'])

    def _get_local_repository_compressed_changelog_file(self, repository_id,
        branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['changelog_filename_compressed'])

    def _get_local_repository_rsslight_file(self, repository_id, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), srv_set['rss']['light_name'])

    def _get_local_repository_notice_board_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['rss-notice-board'])

    def _get_local_repository_treeupdates_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaseupdatefile'])

    def _get_local_repository_compressed_metafiles_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemetafilesfile'])

    def _get_local_repository_metafiles_not_found_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemetafilesnotfound'])

    def _get_local_repository_gpg_signature_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasegpgfile'])

    def _get_local_exp_based_pkgs_rm_whitelist_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaseexpbasedpkgsrm'])

    def _get_local_pkglist_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasepkglist'])

    def _get_local_extra_pkglist_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaseextrapkglist'])

    def _get_local_database_sets_dir(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['confsetsdirname'])

    def _get_local_post_branch_mig_script(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etp_post_branch_hop_script'])

    def _get_local_post_branch_upg_script(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etp_post_branch_upgrade_script'])

    def _get_local_post_repo_update_script(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etp_post_repo_update_script'])

    def _get_local_critical_updates_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasecriticalfile'])

    def _get_local_restricted_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaserestrictedfile'])

    def _get_local_repository_keywords_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasekeywordsfile'])

    def _get_local_repository_webserv_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasewebservicesfile'])

    def _get_local_repository_dir(self, repository_id, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if branch is None:
            branch = self._settings['repositories']['branch']
        return os.path.join(
            srv_set['repositories'][repository_id]['database_dir'], branch)

    def _get_missing_dependencies_blacklist_file(self, repository_id,
        branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if branch is None:
            branch = self._settings['repositories']['branch']
        return os.path.join(
            srv_set['repositories'][repository_id]['database_dir'],
                branch, etpConst['etpdatabasemissingdepsblfile'])

    def _get_repository_lockfile(self, repository_id):
        return os.path.join(self._get_local_repository_dir(repository_id),
            etpConst['etpdatabaselockfile'])

    def _get_repository_download_lockfile(self, repository_id):
        return os.path.join(self._get_local_repository_dir(repository_id),
            etpConst['etpdatabasedownloadlockfile'])

    def _create_local_repository_download_lockfile(self, repository_id):
        lock_file = self._get_repository_download_lockfile(repository_id)
        enc = etpConst['conf_encoding']
        with codecs.open(lock_file, "w", encoding=enc) as f_lock:
            f_lock.write("download locked")

    def _create_local_repository_lockfile(self, repository_id):
        lock_file = self._get_repository_lockfile(repository_id)
        enc = etpConst['conf_encoding']
        with codecs.open(lock_file, "w", encoding=enc) as f_lock:
            f_lock.write("database locked")

    def _remove_local_repository_lockfile(self, repository_id):
        lock_file = self._get_repository_lockfile(repository_id)
        try:
            os.remove(lock_file)
        except OSError:
            pass

    def _remove_local_repository_download_lockfile(self, repository_id):
        lock_file = self._get_repository_download_lockfile(repository_id)
        try:
            os.remove(lock_file)
        except OSError:
            pass

    @staticmethod
    def _get_conf_dep_rewrite_file():
        packages_dir = SystemSettings.packages_config_directory()
        return os.path.join(packages_dir, "packages.server.dep_rewrite")

    @staticmethod
    def _get_conf_dep_blacklist_file():
        packages_dir = SystemSettings.packages_config_directory()
        return os.path.join(packages_dir, "packages.server.dep_blacklist")

    @staticmethod
    def _get_conf_qa_sets_file():
        packages_dir = SystemSettings.packages_config_directory()
        return os.path.join(packages_dir, "packages.server.sets")

    def complete_remote_package_relative_path(self, pkg_rel_url, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(
            srv_set['repositories'][repository_id]['remote_repo_basedir'],
                pkg_rel_url)

    def complete_local_upload_package_path(self, pkg_rel_url, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(
            srv_set['repositories'][repository_id]['upload_basedir'],
                pkg_rel_url)

    def complete_local_package_path(self, pkg_rel_url, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(
            srv_set['repositories'][repository_id]['repo_basedir'],
                pkg_rel_url)

    def remote_repository_mirrors(self, repository_id):
        """
        Return a list of remote repository mirrors (database) for given
        repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: list of available repository mirrors
        @rtype: list
        @raise KeyError: if repository_id is invalid
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['repo_mirrors'][:]

    def remote_packages_mirrors(self, repository_id):
        """
        Return a list of remote packages mirrors (packages) for given
        repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: list of available packages mirrors
        @rtype: list
        @raise KeyError: if repository_id is invalid
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['pkg_mirrors'][:]

    def local_repository_revision(self, repository_id):
        """
        Return local repository revision.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: the actual repository revision
        @rtype: int
        """
        dbrev_file = self._get_local_repository_revision_file(repository_id)
        if not os.path.isfile(dbrev_file):
            return 0

        enc = etpConst['conf_encoding']
        with codecs.open(dbrev_file, "r", encoding=enc) as f_rev:
            rev = f_rev.readline().strip()
        try:
            rev = int(rev)
        except ValueError:
            self.output(
                "[%s] %s: %s - %s" % (
                        darkgreen(repository_id),
                        blue(_("invalid repository revision")),
                        bold(rev),
                        blue(_("defaulting to 0")),
                    ),
                importance = 2,
                level = "error",
                header = darkred(" !!! ")
            )
            rev = 0
        return rev

    def remote_repository_revision(self, repository_id):
        """
        Return the highest repository revision available on mirrors for
        given repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: remote repository revision
        @rtype: int
        """
        repo_status = self.Mirrors.remote_repository_status(repository_id)
        remote_status =  list(repo_status.items())
        if not [x for x in remote_status if x[1]]:
            return 0
        return max([x[1] for x in remote_status])

    def repositories(self):
        """
        Return a list of available Entropy Server repositories.

        @return: list of available Entropy Server repositories
        @rtype: list
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return sorted(srv_set['repositories'])

    def qa_repositories(self):
        """
        Return a list of QA-testable available Entropy Server repositories.

        @return: list of QA-testable available Entropy Server repositories
        @rtype: list
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        repos = srv_set['repositories']
        return sorted([x for x, y in repos.items() if not y['exclude_qa']])

    def repository(self):
        """
        Return the current repository marked as default.
        """
        return self._repository

    def QA(self):
        """
        Get Entropy QA Interface instance.

        @return: Entropy QA Interface instance
        @rtype: entropy.qa.QAInterface instance
        """
        qa_plugin = ServerQAInterfacePlugin(self)
        qa = Client.QA(self)
        qa.add_plugin(qa_plugin)
        return qa

    def Spm(self):
        """
        Get Source Package Manager interface instance.

        #@return: Source Package Manager interface instance
        @rtype: entropy.spm.plugins.skel.SpmPlugin based instance
        """
        return get_spm(self)

    def Spm_class(self):
        """
        Get Source Package Manager interface class.
        """
        return get_spm_class()

    def ConfigurationUpdates(self):
        """
        Return Entropy Configuration File Updates management object.
        """
        return ConfigurationUpdates(
            self, _config_class=ServerConfigurationFiles)

    def Transceiver(self, uri):
        """
        Get EntropyTransceiver interface instance.

        @param uri: EntropyTransceiver URI
        @type uri: string
        @return: EntropyTransceiver instance
        @rtype: entropy.transceivers.EntropyTransceiver
        """
        txc = EntropyTransceiver(uri)
        txc.set_output_interface(self)
        return txc

    def _sync_package_sets(self, entropy_repository):
        """
        Synchronize repository package sets.

        @param entropy_repository: EntropyRepositoryBase object
        @type entropy_repository: entropy.db.skel.EntropyRepositoryBase
        """
        repository_id = entropy_repository.repository_id()
        self.output(
            "[%s|%s] %s" % (
                    blue(repository_id),
                    red(_("repository")),
                    blue(_("syncing package sets")),
                ),
            importance = 1,
            level = "info",
            header = brown(" @@ ")
        )
        cur_sets = entropy_repository.retrievePackageSets()
        sys_sets = self._get_configured_package_sets(repository_id)
        if cur_sets != sys_sets:
            self._update_package_sets(repository_id, entropy_repository)
        # NOTE: this is called by the commit hook plugin, keep no_plugins=True!
        entropy_repository.commit(no_plugins = True)

    def sets_available(self, *args, **kwargs):
        sets = Client.Sets(self)
        return sets.available(*args, **kwargs)

    def sets_search(self, *args, **kwargs):
        sets = Client.Sets(self)
        return sets.search(*args, **kwargs)

    def sets_match(self, *args, **kwargs):
        sets = Client.Sets(self)
        return sets.match(*args, **kwargs)

    def atom_match(self, *args, **kwargs):
        # disable masked packages for server-side repos
        kwargs['mask_filter'] = False
        return Client.atom_match(self, *args, **kwargs)

    def _match_packages(self, repository_id, packages):

        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)
        if ("world" in packages) or not packages:
            return dbconn.listAllPackageIds(), True
        else:
            package_ids = set()
            for package in packages:
                matches = dbconn.atomMatch(package, multiMatch = True)
                if matches[1] == 0:
                    package_ids |= matches[0]
                else:
                    mytxt = "%s: %s: %s" % (
                        red(_("Attention")),
                        blue(_("cannot match")),
                        bold(package),
                    )
                    self.output(
                        mytxt,
                        importance = 1,
                        level = "warning",
                        header = darkred(" !!! ")
                    )
            return package_ids, False

    def mask_packages(self, repository_id, packages):
        """
        Mask given package dependencies for given repository, if any (otherwise
        use default one).

        @param repository_id: repository identifier
        @type repository_id: string
        @param packages: list of package dependency strings
        @type packages: list
        @return: mask status, True if ok, False if not
        @rtype: bool
        """
        mask_file = self._get_local_repository_mask_file(repository_id)
        current_packages = []

        if const_file_readable(mask_file):
            # don't worry about the race.
            current_packages += entropy.tools.generic_file_content_parser(
                mask_file, comment_tag = "##", filter_comments = False,
                encoding = etpConst['conf_encoding'])
        # this is untrusted input, it's fine because that config file is
        # untrusted too
        current_packages.extend(packages)

        mask_file_tmp = mask_file + ".mask_packages_tmp"
        enc = etpConst['conf_encoding']
        with codecs.open(mask_file_tmp, "w", encoding=enc) as mask_f:
            for package in current_packages:
                mask_f.write(package + "\n")

        os.rename(mask_file_tmp, mask_file)

        return True

    def unmask_packages(self, repository_id, packages):
        """
        Unmask given package dependencies for given repository, if any
        (otherwise use default one).

        @param repository_id: repository identifier
        @type repository_id: string
        @param packages: list of package dependency strings
        @type packages: list
        @return: mask status, True if ok, False if not
        @rtype: bool
        """
        mask_file = self._get_local_repository_mask_file(repository_id)
        current_packages = []

        if const_file_readable(mask_file):
            # don't worry about the race.
            current_packages += entropy.tools.generic_file_content_parser(
                mask_file, comment_tag = "##", filter_comments = False,
                encoding = etpConst['conf_encoding'])

        def mask_filter(package):
            if package.startswith("#"):
                # comment, always valid
                return True
            in_file_pkg_match = self.atom_match(package)
            for req_package in packages:
                if package == req_package:
                    # of course remove if it's equal
                    return False
                req_package_match = self.atom_match(req_package)
                if req_package_match == in_file_pkg_match:
                    # drop it, they point to the same package match
                    return False
            return True

        current_packages = list(filter(mask_filter, current_packages))

        mask_file_tmp = mask_file + ".mask_packages_tmp"
        enc = etpConst['conf_encoding']
        with codecs.open(mask_file_tmp, "w", encoding=enc) as mask_f:
            for package in current_packages:
                mask_f.write(package + "\n")

        os.rename(mask_file_tmp, mask_file)

        return True

    def initialize_repository(self, repository_id, ask = True):
        """
        Initialize (and wipe all data!) given repository to empty status.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword ask: ask before making any change?
        @type ask: bool
        @return: execution status (0 = fine)
        @rtype: int
        """
        self.output(
            "[%s] %s..." % (
                purple(repository_id), darkgreen(_("initializing repository")),
            ),
            importance = 1,
            level = "info", header = darkgreen(" * ")
        )
        self.close_repositories()

        rc_question = self.ask_question(
            "[%s] %s" % (
                purple(repository_id),
                teal(_("do you really want to initialize this repository ?"))
            )
        )
        if rc_question == _("No"):
            return 1

        try:
            os.remove(self._get_local_repository_file(repository_id))
        except OSError as err:
            if err.errno != errno.ENOENT:
                raise

        # initialize
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True, is_new = True)
        dbconn.initializeRepository()
        dbconn.commit()

        # create the store directory
        store_dir = self._get_local_store_directory(repository_id)
        if not os.path.isdir(store_dir):
            try:
                os.makedirs(store_dir)
            except (IOError, OSError) as err:
                self.output(
                    "%s: %s" % (_("Cannot create store directory"), err),
                    header=brown(" !!! "),
                    importance=1,
                    level="error")
                return 1

        # create the upload directory
        upload_dir = self._get_local_upload_directory(repository_id)
        if not os.path.isdir(upload_dir):
            try:
                os.makedirs(upload_dir)
            except (IOError, OSError) as err:
                self.output(
                    "%s: %s" % (_("Cannot create upload directory"), err),
                    header=brown(" !!! "),
                    importance=1,
                    level="error")
                return 1

        return 0

    def tag_packages(self, package_matches, package_tag, ask = True):
        """
        Change version tag for given package matches.

        @param package_matches: list of Entropy package matches
        @type package_matches: list
        @param package_tag: new Entropy package tag string
        @type package_tag: string
        @return: execution status (0 = fine)
        @rtype: int
        """
        try:
            package_tag = str(package_tag)
            if " " in package_tag:
                raise ValueError
        except (UnicodeDecodeError, UnicodeEncodeError, ValueError,):
            self.output(
                "%s: %s" % (
                    blue(_("Invalid tag specified")),
                    package_tag,
                ),
                importance = 1, level = "error", header = darkred(" !! ")
            )
            return 1

        pkg_map = {}
        for pkg_id, pkg_repo in package_matches:
            obj = pkg_map.setdefault(pkg_repo, [])
            obj.append(pkg_id)

        for pkg_repo in sorted(pkg_map.keys()):
            switched = self._move_packages(pkg_map[pkg_repo], pkg_repo,
                pkg_repo, ask = ask, do_copy = True, new_tag = package_tag)
            if not switched:
                return 1
        return 0

    def flushback_packages(self, repository_id, from_branches, ask = True):
        """
        When creating a new branch, for space reasons, packages are not
        moved to a new location. This works fine until old branch is removed.
        To avoid inconsistences, before deciding to do that, all the packages
        in the old branch should be flushed back to the the currently configured
        branch.

        @param repository_id: repository identifier
        @type repository_id: string
        @param from_branches: list of branches to move packages from
        @type from_branches: list
        @keyword ask: ask before making any change?
        @type ask: bool
        @return execution status (0 = fine)
        @rtype: int
        """
        branch = self._settings['repositories']['branch']

        if branch in from_branches:
            from_branches = [x for x in from_branches if x != branch]

        self.output(
            "[%s=>%s|%s] %s" % (
                darkgreen(', '.join(from_branches)),
                darkred(branch),
                brown(repository_id),
                blue(_("flushing back selected packages from branches")),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)

        package_id_map = dict(((x, [],) for x in from_branches))
        package_ids = dbconn.listAllPackageIds(order_by = 'atom')
        for package_id in package_ids:
            download_url = dbconn.retrieveDownloadURL(package_id)
            url_br = self._get_branch_from_download_relative_uri(
                download_url)
            if url_br in from_branches:
                package_id_map[url_br].append(package_id)

        mapped_branches = [x for x in package_id_map if package_id_map[x]]
        if not mapped_branches:
            self.output(
                "[%s=>%s|%s] %s !" % (
                    darkgreen(', '.join(from_branches)),
                    darkred(branch),
                    brown(repository_id),
                    blue(_("nothing to do")),
                ),
                importance = 0,
                level = "warning",
                header = blue(" @@ ")
            )
            return 0


        all_fine = True
        tmp_down_dir = const_mkdtemp(prefix="entropy.server")

        download_queue = {}
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)

        def generate_queue(branch, repository_id, from_branch, down_q,
            package_id_map):

            self.output(
                "[%s=>%s|%s] %s" % (
                    darkgreen(from_branch),
                    darkred(branch),
                    brown(repository_id),
                    brown(_("these are the packages that will be flushed")),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )


            for package_id in package_id_map[from_branch]:
                atom = dbconn.retrieveAtom(package_id)
                self.output(
                    "[%s=>%s|%s] %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        purple(atom),
                    ),
                    importance = 0,
                    level = "info",
                    header = blue("  # ")
                )
                pkg_fp = os.path.basename(
                    dbconn.retrieveDownloadURL(package_id))
                pkg_fp = os.path.join(tmp_down_dir, pkg_fp)
                down_q.append((pkg_fp, package_id,))


        for from_branch in sorted(mapped_branches):

            download_queue[from_branch] = []
            all_fine = False
            generate_queue(branch, repository_id, from_branch,
                download_queue[from_branch], package_id_map)

            if ask:
                rc_question = self.ask_question(
                    _("Would you like to continue ?"))
                if rc_question == _("No"):
                    continue

            for uri in self.remote_packages_mirrors(repository_id):

                crippled_uri = EntropyTransceiver.get_uri_name(uri)

                queue_map = {}

                for pkg_fp, package_id in download_queue[from_branch]:
                    down_url = dbconn.retrieveDownloadURL(package_id)
                    down_rel = self.complete_remote_package_relative_path(
                        down_url, repository_id)
                    down_rel_dir = os.path.dirname(down_rel)
                    obj = queue_map.setdefault(down_rel_dir, [])
                    obj.append(pkg_fp)

                errors = False
                m_fine_uris = set()
                m_broken_uris = set()

                for down_rel_dir, downloader_queue in queue_map.items():

                    downloader = self.Mirrors.TransceiverServerHandler(
                        self,
                        [uri],
                        downloader_queue,
                        critical_files = downloader_queue,
                        txc_basedir = down_rel_dir,
                        local_basedir = tmp_down_dir,
                        download = True,
                        repo = repository_id
                    )
                    xerrors, xm_fine_uris, xm_broken_uris = downloader.go()
                    if xerrors:
                        errors = True
                    m_fine_uris.update(xm_fine_uris)
                    m_broken_uris.update(xm_broken_uris)

                if not errors:
                    for downloaded_path, package_id in \
                        download_queue[from_branch]:

                        self.output(
                            "[%s=>%s|%s|%s] %s: %s" % (
                                darkgreen(from_branch),
                                darkred(branch),
                                brown(repository_id),
                                dbconn.retrieveAtom(package_id),
                                blue(_("checking package hash")),
                                darkgreen(os.path.basename(downloaded_path)),
                            ),
                            importance = 0,
                            level = "info",
                            header = brown("   "),
                            back = True
                        )

                        md5hash = entropy.tools.md5sum(downloaded_path)
                        db_md5hash = dbconn.retrieveDigest(package_id)
                        if md5hash != db_md5hash:
                            errors = True
                            self.output(
                                "[%s=>%s|%s|%s] %s: %s" % (
                                    darkgreen(from_branch),
                                    darkred(branch),
                                    brown(repository_id),
                                    dbconn.retrieveAtom(package_id),
                                    blue(_("hash does not match for")),
                                    darkgreen(os.path.basename(downloaded_path)),
                                ),
                                importance = 0,
                                level = "error",
                                header = brown("   ")
                            )
                            continue

                if errors:
                    reason = _("wrong md5")
                    if m_broken_uris:
                        my_broken_uris = [
                        (EntropyTransceiver.get_uri_name(x), y,) \
                            for x, y in m_broken_uris]
                        reason = my_broken_uris[0][1]

                    self.output(
                        "[%s=>%s|%s] %s, %s: %s" % (
                            darkgreen(from_branch),
                            darkred(branch),
                            brown(repository_id),
                            blue(_("download errors")),
                            blue(_("reason")),
                            reason,
                        ),
                        importance = 1,
                        level = "error",
                        header = darkred(" !!! ")
                    )
                    # continuing if possible
                    continue

                all_fine = True

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        blue(_("download completed successfully")),
                        darkgreen(crippled_uri),
                    ),
                    importance = 1,
                    level = "info",
                    header = darkgreen(" * ")
                )

        if not all_fine:
            self.output(
                "[%s=>%s|%s] %s" % (
                    darkgreen(', '.join(from_branches)),
                    darkred(branch),
                    brown(repository_id),
                    blue(_("error downloading packages from mirrors")),
                ),
                importance = 2,
                level = "error",
                header = darkred(" !!! ")
            )
            return 1

        for from_branch in sorted(mapped_branches):

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(from_branch),
                    darkred(branch),
                    brown(repository_id),
                    blue(_("working on branch")),
                    darkgreen(from_branch),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )

            down_queue = download_queue[from_branch]
            for package_path, package_id in down_queue:

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        blue(_("updating package")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = brown("   "),
                    back = True
                )

                # build new download url
                download_url = dbconn.retrieveDownloadURL(package_id)
                download_url = \
                    self._swap_branch_in_download_relative_uri(
                        branch, download_url)

                # move files to upload
                new_package_path = self.complete_local_upload_package_path(
                    download_url, repository_id)
                self._ensure_dir_path(os.path.dirname(new_package_path))

                try:
                    os.rename(package_path, new_package_path)
                except OSError as err:
                    if err.errno != errno.EXDEV:
                        raise
                    shutil.move(package_path, new_package_path)

                # update database
                dbconn.setDownloadURL(package_id, download_url)
                dbconn.commit()
                dbconn.switchBranch(package_id, branch)
                dbconn.commit()

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        blue(_("package flushed")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = brown("   ")
                )

        try:
            os.rmdir(tmp_down_dir)
        except OSError:
            pass

        return 0

    def move_packages(self, package_ids, from_repository_id, to_repository_id,
        ask = True, pull_dependencies = False):
        """
        Move packages from a repository to another.

        @param package_ids: list of package identifiers contained in
            from_repository_id repository
        @type package_ids: list
        @param from_repository_id: source repository identifier
        @type from_repository_id: string
        @param to_repository_id: destination repository identifier
        @type to_repository_id: string
        @keyword ask: execute in interactive mode
        @type ask: bool
        @keyword pull_dependencies: also include reverse dependencies in
            move
        @type pull_dependencies: bool
        @return: list (set) of moved package identifiers
        @rtype: set
        """
        return self._move_packages(package_ids, from_repository_id,
            to_repository_id, ask = ask, pull_deps = pull_dependencies,
            do_copy = False)

    def copy_packages(self, package_ids, from_repository_id, to_repository_id,
        ask = True, pull_dependencies = False):
        """
        Copy packages from a repository to another.

        @param package_ids: list of package identifiers contained in
            from_repository_id repository
        @type package_ids: list
        @param from_repository_id: source repository identifier
        @type from_repository_id: string
        @param to_repository_id: destination repository identifier
        @type to_repository_id: string
        @keyword ask: execute in interactive mode
        @type ask: bool
        @keyword pull_dependencies: also include reverse dependencies in
            copy
        @type pull_dependencies: bool
        @return: list (set) of copied package identifiers
        @rtype: set
        """
        return self._move_packages(package_ids, from_repository_id,
            to_repository_id, ask = ask, pull_deps = pull_dependencies,
            do_copy = True)

    def _move_package(self, package_match, todbconn, new_tag, do_copy):
        """
        Move a single package from a repository to another.

        @param package_match: the package match to move
        @type package_match: tuple
        @param todbconn: the destination EntropyRepository object
        @type todbconn: EntropyRepository
        @param new_tag: a package tag to set on the new package
        @type new_tag: string or None
        @param do_copy: execute copy instead of move
        @type do_copy: bool
        @return: the new package id inside the destination repository or None.
        @rtype: int or None
        """
        branch = self._settings['repositories']['branch']
        package_id, from_repository_id = package_match
        to_repository_id = todbconn.name

        dbconn = self.open_server_repository(
            from_repository_id, read_only = False, no_upload = True)
        match_atom = dbconn.retrieveAtom(package_id)
        package_rel_path = dbconn.retrieveDownloadURL(package_id)

        def _package_injector_check_license(pkg_data):
            licenses = pkg_data['license'].split()
            return self._is_pkg_free(to_repository_id, licenses)

        def _package_injector_check_restricted(pkg_data):
            pkgatom = entropy.dep.create_package_atom_string(
                pkg_data['category'], pkg_data['name'], pkg_data['version'],
                pkg_data['versiontag'])
            return self._is_pkg_restricted(to_repository_id, pkgatom,
                pkg_data['slot'])

        self.output(
            "[%s=>%s|%s] %s: %s" % (
                darkgreen(from_repository_id),
                darkred(to_repository_id),
                brown(branch),
                blue(_("switching")),
                darkgreen(match_atom),
            ),
            importance = 0,
            level = "info",
            header = red(" @@ "),
            back = True
        )
        # move binary file
        from_file = self.complete_local_package_path(package_rel_path,
            from_repository_id)
        if not os.path.isfile(from_file):
            from_file = self.complete_local_upload_package_path(
                package_rel_path, from_repository_id)
        if not os.path.isfile(from_file):
            self.output(
                "[%s=>%s|%s] %s: %s -> %s" % (
                    darkgreen(from_repository_id),
                    darkred(to_repository_id),
                    brown(branch),
                    bold(_("cannot switch, package not found, skipping")),
                    darkgreen(match_atom),
                    red(from_file),
                ),
                importance = 1,
                level = "warning",
                header = darkred(" !!! ")
            )
            return None

        # we need to ask SpmPlugin to re-extract metadata from pkg file
        # and grab the new "download" metadatum value using our
        # license check callback. It has to be done here because
        # we need the new path.

        # check if pkg is restricted
        # and check if pkg is free, we must do this step in any case
        # NOTE: it sucks!
        tmp_data = self.Spm().extract_package_metadata(from_file,
            license_callback = _package_injector_check_license,
            restricted_callback = _package_injector_check_restricted)
        # NOTE: since ~0.tbz2 << revision is lost, we need to trick
        # the logic.
        updated_package_rel_path = os.path.join(
            os.path.dirname(tmp_data['download']),
            os.path.basename(package_rel_path))
        del tmp_data

        to_file = self.complete_local_upload_package_path(
            updated_package_rel_path, to_repository_id)

        if new_tag is not None:

            signatures = dbconn.retrieveSignatures(package_id)
            packge_sha1 = None
            if signatures:
                package_sha1, _ignore, _ignore, _ignore = signatures

            tagged_package_filename = \
                entropy.dep.create_package_filename(
                    dbconn.retrieveCategory(package_id),
                    dbconn.retrieveName(package_id),
                    dbconn.retrieveVersion(package_id),
                    new_tag,
                    revision = dbconn.retrieveRevision(package_id),
                    sha1 = package_sha1)

            to_file = self.complete_local_upload_package_path(
                updated_package_rel_path, to_repository_id)
            # directly move to correct place, tag changed, so file name
            to_file = os.path.join(os.path.dirname(to_file),
                tagged_package_filename)

        self._ensure_dir_path(os.path.dirname(to_file))

        copy_data = [
            (from_file, to_file,),
            (from_file + etpConst['packagesexpirationfileext'],
                to_file + etpConst['packagesexpirationfileext'],)
        ]
        extra_downloads = dbconn.retrieveExtraDownload(package_id)
        for extra_download in extra_downloads:
            # just need the first entry, "download"
            extra_rel = extra_download['download']

            from_extra = self.complete_local_package_path(extra_rel,
                from_repository_id)
            if not os.path.isfile(from_extra):
                from_extra = self.complete_local_upload_package_path(
                    extra_rel, from_repository_id)

            to_extra = self.complete_local_upload_package_path(
                extra_rel, to_repository_id)

            copy_data.append((from_extra, to_extra))

        for from_item, to_item in copy_data:
            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(from_repository_id),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(_("moving file")),
                    darkgreen(os.path.basename(from_item)),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )
            if os.path.isfile(from_item):
                shutil.copy2(from_item, to_item)

        self.output(
            "[%s=>%s|%s] %s: %s" % (
                darkgreen(from_repository_id),
                darkred(to_repository_id),
                brown(branch),
                blue(_("loading data from source repository")),
                darkgreen(from_repository_id),
            ),
            importance = 0,
            level = "info",
            header = red(" @@ "),
            back = True
        )
        # install package into destination db
        data = dbconn.getPackageData(package_id)
        if new_tag != None:
            data['versiontag'] = new_tag

        # need to set back data['download'], because pkg path might got
        # changed, due to license re-validation
        data['download'] = updated_package_rel_path

        # GPG
        # before inserting new pkg, drop GPG signature and re-sign
        old_gpg = copy.copy(data['signatures']['gpg'])
        data['signatures']['gpg'] = None
        for extra_download in data['extra_download']:
            extra_download['gpg'] = None
        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            if old_gpg:
                self.output(
                    "[%s] %s %s: %s." % (
                        darkgreen(to_repository_id),
                        darkred(_("GPG key was available in")),
                        bold(from_repository_id),
                        err,
                    ),
                    importance = 1,
                    level = "warning",
                    header = bold(" !!! ")
                )
            repo_sec = None

        if repo_sec is not None:
            data['signatures']['gpg'] = self._get_gpg_signature(repo_sec,
                to_repository_id, to_file)

            for extra_download in data['extra_download']:
                to_extra = self.complete_local_upload_package_path(
                    extra_download['download'], to_repository_id)
                extra_download['gpg'] = self._get_gpg_signature(repo_sec,
                    to_repository_id, to_extra)

        self.output(
            "[%s=>%s|%s] %s: %s" % (
                darkgreen(from_repository_id),
                darkred(to_repository_id),
                brown(branch),
                blue(_("injecting data to destination repository")),
                darkgreen(to_repository_id),
            ),
            importance = 0,
            level = "info",
            header = red(" @@ "),
            back = True
        )
        data['original_repository'] = to_repository_id
        # force our own revision, to avoid file name collisions
        new_package_id = todbconn.handlePackage(data)
        del data
        todbconn.commit()

        if not do_copy:
            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(from_repository_id),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(_("removing entry from source repository")),
                    darkgreen(from_repository_id),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )

            # remove package from old db
            dbconn.removePackage(package_id)
            dbconn.clean()
            dbconn.commit()

        self.output(
            "[%s=>%s|%s] %s: %s" % (
                darkgreen(from_repository_id),
                darkred(to_repository_id),
                brown(branch),
                blue(_("successfully handled atom")),
                darkgreen(match_atom),
            ),
            importance = 0,
            level = "info",
            header = blue(" @@ ")
        )
        return new_package_id

    def _move_packages(self, package_ids, from_repository_id, to_repository_id,
        ask = True, do_copy = False, new_tag = None, pull_deps = False):
        """
        Move, copy or re-tag packages in repositories.
        If do_copy is True, a copy is executed of package_ids from
        from_repository_id to to_repository_id. If new_tag is not None,
        and from_repository_id == to_repository_id a re-tag will be executed.
        """

        switched = set()
        my_matches = [(x, from_repository_id) for x in package_ids]

        # avoid setting __default__ as default server repo
        if InstalledPackagesRepository.NAME in (to_repository_id,
                                                from_repository_id):
            self.output(
                "%s: %s" % (
                    blue(_("Cannot touch system repository")),
                    red(InstalledPackagesRepository.NAME),
                ),
                importance = 2, level = "warning", header = darkred(" @@ ")
            )
            return switched

        if not my_matches and from_repository_id:
            dbconn = self.open_server_repository(from_repository_id,
                read_only = True, no_upload = True)
            my_matches = set( \
                [(x, from_repository_id) for x in \
                    dbconn.listAllPackageIds()]
            )

        mytxt = _("Preparing to move selected packages to")
        if do_copy:
            mytxt = _("Preparing to copy selected packages to")
        self.output(
            "%s %s:" % (
                blue(mytxt),
                red(to_repository_id),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        self.output(
            "%s: %s" % (
                bold(_("Note")),
                red(_("all old packages with conflicting scope will be " \
                    "removed from destination repo unless injected")),
            ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        new_tag_string = ''
        if new_tag != None:
            new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),
                brown(new_tag),)

        # open both repos here to make sure it's all fine with them
        dbconn = self.open_server_repository(from_repository_id,
            read_only = True, no_upload = True)
        todbconn = self.open_server_repository(to_repository_id,
            read_only = False, no_upload = True)

        my_qa = self.QA()
        branch = self._settings['repositories']['branch']
        pull_deps_matches = []
        for package_id, repo in my_matches:
            dbconn = self.open_server_repository(repo, read_only = True,
                no_upload = True)
            self.output(
                "[%s=>%s|%s] %s " % (
                    darkgreen(repo),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(dbconn.retrieveAtom(package_id)),
                ) + new_tag_string,
                importance = 0,
                level = "info",
                header = brown("    # ")
            )

            # check if there are pkgs that are going to be overwritten
            # and warn user about that.
            from_name, from_category, from_slot, from_injected = \
                dbconn.retrieveName(package_id), \
                dbconn.retrieveCategory(package_id), \
                dbconn.retrieveSlot(package_id), \
                dbconn.isInjected(package_id)
            to_rm_package_ids = todbconn.getPackagesToRemove(from_name,
                from_category, from_slot, from_injected)
            for to_rm_package_id in to_rm_package_ids:
                self.output(
                    "    [=>%s|%s] %s" % (
                            darkred(to_repository_id),
                            bold(_("remove")),
                            blue(todbconn.retrieveAtom(to_rm_package_id)),
                    ),
                    importance = 0,
                    level = "info",
                    header = purple("    # ")
                )

            # do we want to pull in also package dependencies?
            if pull_deps:
                dep_matches = my_qa.get_deep_dependency_list(self,
                    (package_id, repo), match_repo = (repo,))
                revdep_matches = self.get_reverse_queue(dep_matches,
                    system_packages = False)
                dep_matches.update(revdep_matches)

                for dep_package_id, dep_repo in dep_matches:

                    my_dep_match = (dep_package_id, dep_repo,)
                    if my_dep_match in pull_deps_matches:
                        continue
                    if my_dep_match in my_matches:
                        continue

                    pull_deps_matches.append(my_dep_match)
                    dep_dbconn = self.open_server_repository(dep_repo,
                        read_only = True, no_upload = True)
                    dep_atom = dep_dbconn.retrieveAtom(dep_package_id)
                    if my_dep_match in revdep_matches:
                        self.output(
                            "[%s|%s] %s" % (
                                brown(branch),
                                blue(_("reverse dependency")),
                                teal(dep_atom),
                            ),
                            importance = 0,
                            level = "info",
                            header = purple("    >> ")
                        )
                    else:
                        self.output(
                            "[%s|%s] %s" % (
                                brown(branch),
                                blue(_("dependency")),
                                purple(dep_atom),
                            ),
                            importance = 0,
                            level = "info",
                            header = purple("    >> ")
                        )

        if pull_deps:
            # put deps first!
            my_matches = pull_deps_matches + [x for x in my_matches if x not \
                in pull_deps_matches]

        if ask:
            rc_question = self.ask_question(_("Would you like to continue ?"))
            if rc_question == _("No"):
                return switched

        package_ids_added = set()
        for s_package_id, s_repository_id in my_matches:
            new_package_id = self._move_package(
                (s_package_id, s_repository_id), todbconn,
                new_tag, do_copy)
            if new_package_id is not None:
                switched.add(s_package_id)
                package_ids_added.add(new_package_id)

        todbconn = self.open_server_repository(to_repository_id,
            read_only = False, no_upload = True)
        todbconn.clean()

        if package_ids_added:
            self._add_packages_qa_tests(
                [(x, to_repository_id) for x in package_ids_added], ask = ask)
            # just run this to make dev aware
            self.extended_dependencies_test([to_repository_id])

        return switched

    def _inject_database_into_packages(self, repository_id, injection_data):

        self.output(
            "[%s] %s:" % (
                darkgreen(repository_id),
                blue(_("Injecting entropy metadata into built packages")),
            ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            self.output(
                "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("JFYI, GPG infrastructure failed to load")),
                    err,
                ),
                importance = 1,
                level = "warning",
                header = red(" @@ ")
            )
            repo_sec = None # gnupg not found, perhaps report it

        treeupdates_actions = dbconn.listAllTreeUpdatesActions()

        # generate empty repository file and re-use it every time
        # this improves the execution a lot
        orig_fd = None
        tmp_repo_orig_path = None
        try:
            orig_fd, tmp_repo_orig_path = const_mkstemp(
                prefix="entropy.server._inject")

            empty_repo = GenericRepository(
                readOnly = False,
                dbFile = tmp_repo_orig_path,
                name = None,
                xcache = False,
                indexing = False,
                skipChecks = True)
            empty_repo.initializeRepository()
            empty_repo.bumpTreeUpdatesActions(treeupdates_actions)
            empty_repo.commit()
        except Exception:
            os.remove(tmp_repo_orig_path)
            raise
        finally:
            if empty_repo is not None:
                empty_repo.close()
            if orig_fd is not None:
                os.close(orig_fd)

        try:
            for package_id, package_path in injection_data:

                tmp_repo_file = None
                tmp_fd = None
                try:
                    tmp_fd, tmp_repo_file = const_mkstemp(
                        prefix="entropy.server._inject_for")
                    with os.fdopen(tmp_fd, "wb") as tmp_f:
                        with open(tmp_repo_orig_path, "rb") as empty_f:
                            shutil.copyfileobj(empty_f, tmp_f)

                    self.output(
                        "[%s|%s] %s: %s" % (
                            darkgreen(repository_id),
                            brown(str(package_id)),
                            blue(_("injecting entropy metadata")),
                            darkgreen(os.path.basename(package_path)),
                        ),
                        importance = 1,
                        level = "info",
                        header = blue(" @@ "),
                        back = True
                    )
                    data = dbconn.getPackageData(package_id)
                    self._inject_entropy_database_into_package(
                        package_path, data,
                        treeupdates_actions = treeupdates_actions,
                        initialized_repository_path = tmp_repo_file)
                finally:
                    if tmp_fd is not None:
                        try:
                            os.close(tmp_fd)
                        except OSError as err:
                            if err.errno != errno.EBADF:
                                raise
                    if tmp_repo_file is not None:
                        os.remove(tmp_repo_file)

                # GPG-sign package if GPG signature is set
                gpg_sign = None
                if repo_sec is not None:
                    gpg_sign = self._get_gpg_signature(repo_sec, repository_id,
                        package_path)

                digest = entropy.tools.md5sum(package_path)
                # update digest
                dbconn.setDigest(package_id, digest)
                # update signatures
                signatures = data['signatures'].copy()
                for hash_key in sorted(signatures):
                    if hash_key == "gpg": # gpg already created
                        continue
                    hash_func = getattr(entropy.tools, hash_key)
                    signatures[hash_key] = hash_func(package_path)
                dbconn.setSignatures(package_id, signatures['sha1'],
                    signatures['sha256'], signatures['sha512'],
                    gpg_sign)

                # recompute the package file name and download url
                # to match the final SHA1.
                download_url = self._setup_repository_package_filename(
                    dbconn, package_id)
                package_dir = os.path.dirname(package_path)
                new_package_path = os.path.join(
                    package_dir, os.path.basename(download_url))
                os.rename(package_path, new_package_path)
                package_path = new_package_path

                dbconn.commit()

                const_setup_file(package_path, etpConst['entropygid'], 0o664)
                self.output(
                    "[%s|%s] %s: %s" % (
                        darkgreen(repository_id),
                        brown(str(package_id)),
                        blue(_("injection complete")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = red(" @@ ")
                )
        finally:
            os.remove(tmp_repo_orig_path)

    def remove_packages(self, repository_id, package_ids):
        """
        Remove packages from given repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @param package_ids: list of package identifiers contained in
            from_repository_id repository
        @type package_ids: list
        """
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        for package_id in package_ids:
            atom = dbconn.retrieveAtom(package_id)
            self.output(
                "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("removing package")),
                    darkgreen(atom),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            dbconn.removePackage(package_id)
        dbconn.clean()
        dbconn.commit()
        self.close_repository(dbconn)
        self.output(
            "[%s] %s" % (
                darkgreen(repository_id),
                blue(_("removal complete")),
            ),
            importance = 1,
            level = "info",
            header = brown(" @@ ")
        )

    def _verify_remote_packages(self, repository_id, packages, ask = True):

        self.output(
            "[%s] %s:" % (
                red("remote"),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ ")
        )

        package_ids, world = self._match_packages(repository_id, packages)
        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)
        branch = self._settings['repositories']['branch']

        if world:
            self.output(
                blue(
                    _("All the packages in repository will be checked.")),
                importance = 1,
                level = "info",
                header = "    "
            )
        else:
            mytxt = red("%s:") % (
                _("This is the list of the packages that would be checked"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = "    "
            )
            for package_id in package_ids:
                pkgatom = dbconn.retrieveAtom(package_id)
                down_url = dbconn.retrieveDownloadURL(package_id)
                pkgfile = os.path.basename(down_url)
                self.output(
                    red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)),
                    importance = 1,
                    level = "info",
                    header = darkgreen("   - ")
                )

        if ask:
            rc_question = self.ask_question(
                _("Would you like to continue ?"))
            if rc_question == _("No"):
                return set(), set(), {}

        match = set()
        not_match = set()
        broken_packages = {}

        for uri in self.remote_packages_mirrors(repository_id):

            crippled_uri = EntropyTransceiver.get_uri_name(uri)
            self.output(
                "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("Working on mirror")),
                    brown(crippled_uri),
                ),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )


            totalcounter = len(package_ids)
            currentcounter = 0

            txc = self.Transceiver(uri)
            txc.set_verbosity(False)
            with txc as handler:

                for package_id in package_ids:

                    currentcounter += 1
                    pkgfile = dbconn.retrieveDownloadURL(package_id)
                    pkgfile = self.complete_remote_package_relative_path(
                        pkgfile, repository_id)
                    pkghash = dbconn.retrieveDigest(package_id)

                    self.output(
                        "[%s] %s: %s" % (
                            brown(crippled_uri),
                            blue(_("checking hash")),
                            darkgreen(pkgfile),
                        ),
                        importance = 1,
                        level = "info",
                        header = blue(" @@ "),
                        back = True,
                        count = (currentcounter, totalcounter,)
                    )

                    ck_remote = handler.get_md5(pkgfile)
                    if ck_remote is None:
                        self.output(
                            "[%s] %s: %s %s" % (
                                brown(crippled_uri),
                                blue(_("digest verification of")),
                                bold(pkgfile),
                                blue(_("not supported")),
                            ),
                            importance = 1,
                            level = "info",
                            header = blue(" @@ "),
                            count = (currentcounter, totalcounter,)
                        )
                        continue

                    if ck_remote == pkghash:
                        match.add(package_id)
                    else:
                        not_match.add(package_id)
                        self.output(
                            "[%s] %s: %s %s" % (
                                brown(crippled_uri),
                                blue(_("package")),
                                bold(pkgfile),
                                red(_("NOT healthy")),
                            ),
                            importance = 1,
                            level = "warning",
                            header = darkred(" !!! "),
                            count = (currentcounter, totalcounter,)
                        )
                        if crippled_uri not in broken_packages:
                            broken_packages[crippled_uri] = []
                        broken_packages[crippled_uri].append(pkgfile)

            if broken_packages:
                mytxt = blue("%s:") % (
                    _("This is the list of broken packages"),)
                self.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = red(" * ")
                )
                for mirror in list(broken_packages.keys()):
                    mytxt = "%s: %s" % (
                        brown(_("Mirror")),
                        bold(mirror),
                    )
                    self.output(
                        mytxt,
                        importance = 1,
                        level = "info",
                        header = red("   <> ")
                    )
                    for broken_package in broken_packages[mirror]:
                        self.output(
                            blue(broken_package),
                            importance = 1,
                            level = "info",
                            header = red("      - ")
                        )

            self.output(
                "%s:" % (
                    blue(_("Statistics")),
                ),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    brown(_("Number of checked packages")),
                    brown(str(len(match) + len(not_match))),
                ),
                importance = 1,
                level = "info",
               header = brown("   # ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    darkgreen(_("Number of healthy packages")),
                    darkgreen(str(len(match))),
                ),
                importance = 1,
                level = "info",
               header = brown("   # ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    darkred(_("Number of broken packages")),
                    darkred(str(len(not_match))),
                ),
                importance = 1,
                level = "info",
                header = brown("   # ")
            )

        return match, not_match, broken_packages

    def _verify_local_packages(self, repository_id, packages, ask = True):

        self.output(
            "[%s] %s:" % (
                red(_("local")),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

        package_ids, world = self._match_packages(repository_id, packages)
        dbconn = self.open_server_repository(repository_id, read_only = True)
        if world:
            self.output(
                blue(_("All the packages in repository will be checked.")),
                importance = 1,
                level = "info",
                header = "    "
            )

        fine = set()
        failed = set()

        rc_status, available, downloaded_fine, downloaded_errors = \
            self._download_locally_missing_files(package_ids, repository_id,
                ask = ask)
        if not rc_status:
            return fine, failed, downloaded_fine, downloaded_errors

        my_qa = self.QA()

        totalcounter = str(len(available))
        currentcounter = 0
        for package_id in available:
            currentcounter += 1
            pkg_path = dbconn.retrieveDownloadURL(package_id)

            self.output(
                "%s: %s" % (
                    blue(_("checking status of")),
                    darkgreen(pkg_path),
                ),
                importance = 1,
                level = "info",
                header = "   ",
                back = True,
                count = (currentcounter, totalcounter,)
            )

            storedmd5 = dbconn.retrieveDigest(package_id)
            pkgpath = self._get_package_path(repository_id, dbconn, package_id)
            result = entropy.tools.compare_md5(pkgpath, storedmd5)
            qa_fine = my_qa.entropy_package_checks(pkgpath)
            if result and qa_fine:
                fine.add(package_id)
            else:
                failed.add(package_id)
                self.output(
                    "%s: %s -- %s: %s" % (
                            blue(_("package")),
                            darkgreen(pkg_path),
                            blue(_("is corrupted, stored checksum")),
                            brown(storedmd5),
                    ),
                    importance = 1,
                    level = "info",
                    header = "   ",
                    count = (currentcounter, totalcounter,)
                )

        if failed:
            mytxt = blue("%s:") % (_("This is the list of broken packages"),)
            self.output(
                    mytxt,
                    importance = 1,
                    level = "warning",
                    header =  darkred("  # ")
            )
            for package_id in failed:
                atom = dbconn.retrieveAtom(package_id)
                down_p = dbconn.retrieveDownloadURL(package_id)
                self.output(
                        blue("[atom:%s] %s" % (atom, down_p,)),
                        importance = 0,
                        level = "warning",
                        header =  brown("    # ")
                )

        # print stats
        self.output(
            red("Statistics:"),
            importance = 1,
            level = "info",
            header = blue(" * ")
        )
        self.output(
            brown("%s => %s" % (
                    len(fine) + len(failed),
                    _("checked packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkgreen("%s => %s" % (
                    len(fine),
                    _("healthy packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkred("%s => %s" % (
                    len(failed),
                    _("broken packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            blue("%s => %s" % (
                    len(downloaded_fine),
                    _("downloaded packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            bold("%s => %s" % (
                    len(downloaded_errors),
                    _("failed downloads"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )

        self.close_repository(dbconn)
        return fine, failed, downloaded_fine, downloaded_errors

    def sign_local_packages(self, repository_id, ask = True):
        """
        Sign local packages in given repository using GPG key hopefully set
        for it.

        @raises OnlineMirrorError: if package path is not available after
            having tried to download it, this should never happen btw.
        """
        self.output(
            "[%s] %s: %s" % (
                red(_("local")),
                blue(_("GPG signing packages for repository")),
                repository_id,
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = False)
        package_ids = dbconn.listAllPackageIds()

        self.output(
            blue(_("All the missing packages in repository will be downloaded.")),
            importance = 1,
            level = "info",
            header = "    "
        )

        rc_status, available, downloaded_fine, downloaded_errors = \
            self._download_locally_missing_files(package_ids, repository_id,
                ask = ask)
        if not rc_status:
            return False, 0, 0

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            self.output("%s: %s" % (
                    darkgreen(_("GnuPG not available")),
                    err,
                ),
                level = "error"
            )
            return False, 0, 0

        kp_expired = False
        try:
            kp_avail = repo_sec.is_keypair_available(repository_id)
        except RepositorySecurity.KeyExpired:
            kp_avail = False
            kp_expired = True

        if kp_expired:
            for x in (1, 2, 3):
                # SPAM!
                self.output("%s: %s" % (
                        darkred(_("Keys for repository are expired")),
                        bold(repository_id),
                    ),
                    level = "warning",
                    header = bold(" !!! ")
                )
        elif not kp_avail:
            self.output("%s: %s" % (
                    darkgreen(_("Keys not available for")),
                    bold(repository_id),
                ),
                level = "error"
            )
            return False, 0, 0

        fine = 0
        failed = 0
        totalcounter = len(available)
        currentcounter = 0

        # clear all GPG signatures?

        # we can eventually sign!
        for package_id in available:

            currentcounter += 1

            pkg_path = self._get_package_path(repository_id, dbconn, package_id)
            if not os.path.isfile(pkg_path):
                pkg_path = self._get_upload_package_path(repository_id, dbconn,
                    package_id)
            if not os.path.isfile(pkg_path):
                # wtf!?
                pkg_atom = dbconn.retrieveAtom(package_id)
                raise OnlineMirrorError("WTF!?!?! => %s, %s" % (
                    pkg_path, pkg_atom,))

            self.output(
                "%s: %s" % (
                    blue(_("signing package")),
                    darkgreen(os.path.basename(pkg_path)),
                ),
                importance = 1,
                level = "info",
                header = "   ",
                back = True,
                count = (currentcounter, totalcounter,)
            )

            gpg_sign = self._get_gpg_signature(repo_sec, repository_id,
                pkg_path)
            if gpg_sign is None:
                self.output(
                    "%s: %s" % (
                        darkred(_("Unknown error signing package")),
                        darkgreen(os.path.basename(pkg_path)),
                    ),
                    importance = 1,
                    level = "error",
                    header = "   ",
                    count = (currentcounter, totalcounter,)
                )
                failed += 1
                continue

            # now inject gpg signature into repo
            sha1, sha256, sha512, old_gpg = dbconn.retrieveSignatures(
                package_id)
            dbconn.setSignatures(package_id, sha1, sha256, sha512,
                gpg = gpg_sign)

            fine += 1

        dbconn.commit()

        # print stats
        self.output(
            red("Statistics:"),
            importance = 1,
            level = "info",
            header = blue(" * ")
        )
        self.output(
            brown("%s => %s" % (
                    fine,
                    _("signed packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkred("%s => %s" % (
                    failed,
                    _("broken packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            blue("%s => %s" % (
                    len(downloaded_fine),
                    _("downloaded packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            bold("%s => %s" % (
                    len(downloaded_errors),
                    _("failed downloads"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )

        self.close_repository(dbconn)
        return True, fine, failed

    def _download_locally_missing_files(self, package_ids, repository_id,
        ask = True):

        dbconn = self.open_server_repository(repository_id, read_only = True)
        to_download = set()
        available = set()
        for package_id in package_ids:

            bindir_path = self._get_package_path(repository_id, dbconn,
                package_id)
            uploaddir_path = self._get_upload_package_path(repository_id,
                dbconn, package_id)
            pkg_path = dbconn.retrieveDownloadURL(package_id)

            pkgatom = dbconn.retrieveAtom(package_id)
            if os.path.isfile(bindir_path):
                self.output(
                    "[%s] %s :: %s" % (
                        darkgreen(_("available")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
                available.add(package_id)
            elif os.path.isfile(uploaddir_path):
                self.output(
                    "[%s] %s :: %s" % (
                        darkred(_("upload/ignored")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
            else:
                self.output(
                    "[%s] %s :: %s" % (
                        brown(_("download")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
                to_download.add((package_id, pkg_path,))

        if to_download and ask:
            rc_question = self.ask_question(_("Would you like to continue ?"))
            if rc_question == _("No"):
                # = downloaded fine, downloaded error
                return False, available, set(), set()

        if not to_download:
            # = downloaded fine, downloaded error
            return True, available, set(), set()

        downloaded_fine = set()
        downloaded_errors = set()

        not_downloaded = set()
        mytxt = blue("%s ...") % (_("Starting to download missing files"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = "   "
        )
        for uri in self.remote_packages_mirrors(repository_id):

            if not_downloaded:
                mytxt = blue("%s ...") % (
                    _("Searching missing/broken files on another mirror"),)
                self.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = "   "
                )
                to_download = not_downloaded.copy()
                not_downloaded = set()

            for package_id, pkg_path in to_download:
                rc_down = self.Mirrors.download_package(repository_id, uri,
                    pkg_path)
                if rc_down:
                    downloaded_fine.add(package_id)
                    available.add(package_id)
                else:
                    not_downloaded.add(pkg_path)

            if not not_downloaded:
                self.output(
                    red(_("Binary packages downloaded successfully.")),
                    importance = 1,
                    level = "info",
                    header = "   "
                )
                break

        if not_downloaded:
            mytxt = blue("%s:") % (
                _("These are the packages that cannot be found online"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = "   "
            )
            for pkg_path in not_downloaded:
                downloaded_errors.add(pkg_path)
                self.output(
                        brown(pkg_path),
                        importance = 1,
                        level = "warning",
                        header = red("    * ")
                )
            downloaded_errors |= not_downloaded
            mytxt = "%s." % (_("They won't be checked"),)
            self.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "   "
            )

        return True, available, downloaded_fine, downloaded_errors

    def _switch_packages_branch(self, repository_id, from_branch, to_branch):

        if to_branch != self._settings['repositories']['branch']:
            mytxt = "%s: %s %s" % (
                blue(_("Please setup your branch to")),
                bold(to_branch),
                blue(_("and retry")),
            )
            self.output(
                mytxt,
                importance = 1,
                level = "error",
                header = darkred(" !! ")
            )
            return None

        mytxt = red("%s ...") % (_("Copying repository (if not exists)"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )
        branch_dbdir = self._get_local_repository_dir(repository_id)
        old_branch_dbdir = self._get_local_repository_dir(repository_id,
            branch = from_branch)

        # close all our databases
        self.close_repositories()

        # if database file did not exist got created as an empty file
        # we can just rm -rf it
        branch_dbfile = self._get_local_repository_file(repository_id)
        if os.path.isfile(branch_dbfile):
            if entropy.tools.get_file_size(branch_dbfile) == 0:
                shutil.rmtree(branch_dbdir, True)

        if os.path.isdir(branch_dbdir):

            while True:
                rnd_num = entropy.tools.get_random_number()
                backup_dbdir = branch_dbdir + str(rnd_num)
                if not os.path.isdir(backup_dbdir):
                    break
            os.rename(branch_dbdir, backup_dbdir)

        if os.path.isdir(old_branch_dbdir):
            shutil.copytree(old_branch_dbdir, branch_dbdir)

        mytxt = red("%s ...") % (_("Switching packages"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True, lock_remote = False)
        try:
            dbconn.validate()
        except SystemDatabaseError:
            self._handle_uninitialized_repository(repository_id)
            dbconn = self.open_server_repository(repository_id,
                read_only = False, no_upload = True, lock_remote = False)

        package_ids = dbconn.listAllPackageIds()
        already_switched = set()
        not_found = set()
        switched = set()
        ignored = set()
        no_checksum = set()

        maxcount = len(package_ids)
        count = 0
        for package_id in package_ids:
            count += 1

            cur_branch = dbconn.retrieveBranch(package_id)
            atom = dbconn.retrieveAtom(package_id)
            if cur_branch == to_branch:
                already_switched.add(package_id)
                self.output(
                    red("%s %s, %s %s" % (
                            _("Ignoring"),
                            bold(atom),
                            _("already in branch"),
                            cur_branch,
                        )
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen(" @@ "),
                    count = (count, maxcount,)
                )
                ignored.add(package_id)
                continue

            self.output(
                "[%s=>%s] %s" % (
                    brown(cur_branch),
                    bold(to_branch),
                    darkgreen(atom),
                ),
                importance = 0,
                level = "info",
                header = darkgreen(" @@ "),
                back = True,
                count = (count, maxcount,)
            )
            dbconn.switchBranch(package_id, to_branch)
            dbconn.commit()
            switched.add(package_id)

        dbconn.commit()

        # now migrate counters
        dbconn.moveSpmUidsToBranch(to_branch)

        self.close_repository(dbconn)
        mytxt = blue("%s.") % (_("migration loop completed"),)
        self.output(
            "[%s=>%s] %s" % (
                    brown(from_branch),
                    bold(to_branch),
                    mytxt,
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

        return switched, already_switched, ignored, not_found, no_checksum


    def orphaned_spm_packages_test(self):

        mytxt = "%s %s" % (
            blue(_("Running orphaned SPM packages test")), red("..."),)
        self.output(
            mytxt,
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        installed_packages = self.Spm().get_installed_packages()
        length = len(installed_packages)
        installed_packages.sort()
        not_found = {}
        count = 0
        for installed_package in installed_packages:
            count += 1
            self.output(
                "%s: %s" % (
                    darkgreen(_("Scanning package")),
                    brown(installed_package),),
                importance = 0,
                level = "info",
                back = True,
                count = (count, length),
                header = darkred(" @@ ")
            )
            key = entropy.dep.dep_getkey(installed_package)
            try:
                slot = self.Spm().get_installed_package_metadata(
                    installed_package, "SLOT")
            except KeyError:
                # ignore, race condition, pkg got removed in the meantime
                continue
            pkg_atom = "%s%s%s" % (key, ":", slot,)
            try:
                tree_atoms = self.Spm().match_package(pkg_atom,
                    match_type = "match-all")
            except (KeyError,):
                tree_atoms = None # ouch!
            if not tree_atoms:
                not_found[installed_package] = pkg_atom
                self.output(
                    "%s: %s" % (
                        blue(pkg_atom),
                        darkred(_("not found anymore")),
                    ),
                    importance = 0,
                    level = "warning",
                    count = (count, length),
                    header = darkred(" @@ ")
                )

        if not_found:
            not_found_list = ' '.join([not_found[x] for x in sorted(not_found)])
            self.output(
                "%s: %s" % (
                        blue(_("Packages string")),
                        not_found_list,
                    ),
                importance = 0,
                level = "warning",
                count = (count, length),
                header = darkred(" @@ ")
            )

        return not_found

    def _deps_tester(self, default_repository_id, match_repo = None):

        repository_ids = self.repositories()
        if match_repo is None:
            match_repo = repository_ids

        # if a default repository is passed, we will just test against it
        if default_repository_id:
            repository_ids = [default_repository_id]

        deps_not_satisfied = set()
        txt = _("scanning dependencies")

        for repository_id in repository_ids:
            repo = self.open_repository(repository_id)
            dependencies = repo.listAllDependencies()

            total = len(dependencies)
            for count, (dep_id, dep) in enumerate(dependencies, 1):

                if (count % 150 == 0) or (count == total) or (count == 1):
                    self.output(
                        "[%s] %s" % (
                            purple(repository_id),
                            darkgreen(txt),),
                        importance = 0,
                        level = "info",
                        back = True,
                        count = (count, total),
                        header = darkred(" @@ ")
                    )

                pkg_id, _pkg_repo = self.atom_match(
                    dep, match_repo = match_repo)
                if pkg_id == -1:
                    # only if the dependency string is still valid
                    if repo.searchPackageIdFromDependencyId(dep_id):
                        deps_not_satisfied.add(dep)

        return deps_not_satisfied

    def _drained_dependencies_test_scan(self, merged, drained,
                                        use_cache = True):
        """
        See drained_dependencies_test() for more information.
        This method handles an individual case generated by it.

        @param merged: the list of repositories proposed for merge
        @type merged: list
        @param drained: the list of repositories proposed for drain
        @type drained: list
        @keyword use_cache: use on-disk cache
        @type use_cache: bool
        @return: missing dependencies dict
        @rtype: dict
        """
        spm = self.Spm()
        cached = None
        cache_key = None
        if use_cache:

            checksum_m = []
            checksum_d = []
            repo_ck = {}

            for repository_id in merged:
                repo = self.open_repository(repository_id)
                ck = repo.checksum(include_dependencies = True)
                repo_ck[repository_id] = ck
                checksum_m.append(ck)

            for repository_id in drained:
                ck = repo_ck.get(repository_id)
                if ck is None:
                    repo = self.open_repository(repository_id)
                    ck = repo.checksum(include_dependencies = True)
                checksum_d.append(ck)

            c_hash = "%s|%s~%s|%s|%f|v4" % (
                ",".join(merged),
                ",".join(checksum_m),
                ",".join(drained),
                ",".join(checksum_d),
                spm.installed_mtime(),
            )
            sha = hashlib.sha1(const_convert_to_rawstring(c_hash))

            cache_key = "%s/%s" % (
                self._cache_prefix("drained_dependencies_test"),
                sha.hexdigest())
            cached = self._cacher.pop(cache_key)
            if cached is not None:
                return cached

        outcome = {}
        deps_cache = set()  # used for memoization
        for repository_id in merged:

            repo = self.open_repository(repository_id)

            package_ids = repo.listAllPackageIds()
            total = len(package_ids)
            missing = {}
            for count, package_id in enumerate(package_ids, 1):

                if count in (0, total) or count % 150 == 0:
                    mytxt = "%s: %s" % (
                        darkgreen(_("checking repository")),
                        purple(repository_id))
                    self.output(mytxt, header = blue("::"),
                                count = (count, total),
                                back = True)

                xdeps = repo.retrieveDependencies(package_id)
                xdeps = [x for x in xdeps if x not in deps_cache]
                deps_cache.update(xdeps)

                for dependency in xdeps:

                    # need to check inside merged.
                    pkg_id, pkg_repo = self.atom_match(
                        dependency, match_repo=merged)
                    if pkg_id == -1:
                        # potentially broken candidate, but this is
                        # detected by typical dep testing.
                        continue

                    pkg_keyslot = self.open_repository(
                        pkg_repo).retrieveKeySlotAggregated(pkg_id)
                    # match keyslot inside drained, if there is something
                    # we check with dependency.
                    drained_pkg_id, drained_repo = self.atom_match(
                        pkg_keyslot, match_repo=drained)
                    if drained_pkg_id == -1:
                        # nothing, all good
                        continue

                    # then match with dependency.
                    drained_pkg_id, drained_repo = self.atom_match(
                        dependency, match_repo=drained)
                    if drained_pkg_id != -1:
                        # all good then, we are still able to match
                        # a dependency there.
                        continue

                    # in this case, we need to check if the top level
                    # package match (package_id, repository_id) is going
                    # away. If it does, then there is no need to worry.
                    package_keyslot = repo.retrieveKeySlotAggregated(
                        package_id)
                    pkg_id, repo_id = self.atom_match(
                        package_keyslot, match_repo=drained)
                    if pkg_id != -1:
                        continue

                    # check if the package is still installed on the system
                    package_atom = spm.convert_from_entropy_package_name(
                        repo.retrieveAtom(package_id))
                    inst_matches = spm.match_installed_package(
                        package_atom)
                    if not inst_matches:
                        continue
                    # missing dependency!
                    obj = missing.setdefault(package_id, set())
                    obj.add(dependency)

            outcome[repository_id] = missing

        if use_cache and cache_key is not None:
            self._cacher.push(cache_key, outcome)

        return outcome

    def drained_dependencies_test(self, repository_ids, use_cache = True):
        """
        Test repositories against missing dependencies taking into
        consideration the possibility of repositories being drained
        and their content merged into the repositories before them.
        So, if you have 3 repositories, say: main, limbo, hell, this
        method will test:
        - hell drained and merged into main and limbo
        - limbo and hell drained and merged into main

        @param repository_ids: ordered list of repository identifiers
            to test, like the output of Server.repositories()
        @type repository_ids: list
        @keyword use_cache: use on-disk cache
        @type use_cache: bool
        @return: list (set) of unsatisfied dependencies
        @rtype: set
        """
        repos = list(reversed(repository_ids))
        missing_dependencies = set()
        cases = []
        for case in reversed(range(len(repos) - 1)):
            cases.append((repos[0:case+1], repos[case+1:]))

        for merged, drained in cases:

            mytxt = "%s..." % (
                teal(_("Calculating dependencies")),)
            self.output(
                mytxt,
                importance = 2,
                level = "info",
                header = brown(" @@ ")
            )
            mytxt = "%s: %s" % (
                blue(_("if these repositories were merged")),
                ", ".join([darkgreen(x) for x in merged]))
            self.output(
                mytxt,
                header = "   "
            )
            mytxt = "%s: %s" % (
                blue(_("and if these repositories were drained")),
                ", ".join([purple(x) for x in drained]))
            self.output(
                mytxt,
                header = "   "
            )

            # merged, drained, repos
            data = self._drained_dependencies_test_scan(
                merged, drained, use_cache = use_cache)

            for repo_id, m_data in data.items():
                for pkg_id, deps in m_data.items():
                    missing_dependencies.update(deps)

                    p_repo = self.open_repository(repo_id)
                    atom = p_repo.retrieveAtom(pkg_id)
                    mytxt = "[%s] %s, %s:" % (
                        brown(repo_id),
                        teal(atom),
                        darkgreen(_("missing dependencies")))
                    self.output(mytxt, header = " @@ ")

                    for dependency in sorted(deps):
                        self.output(darkgreen(dependency),
                                    header = "   - ")

        return missing_dependencies

    def injected_library_dependencies_test(self, repository_ids,
                                           use_cache = True):
        """
        Test repositories against missing library dependencies
        for injected packages.
        Injected packages are not actually tracking any specific
        package on the system, thus they need particular attention
        in terms of library-level dependencies.

        @param repository_ids: ordered list of repository identifiers
            to test, like the output of Server.repositories()
        @type repository_ids: list
        @keyword use_cache: use on-disk cache
        @type use_cache: bool
        @return: list (set) of package matches with broken dependencies
        @rtype: set
        """
        missing_map = None
        cached = None
        cache_key = None
        # check if result is cached
        if use_cache:

            sorted_r = sorted(repository_ids)
            checksum_r = []
            for repository_id in sorted_r:
                repo = self.open_repository(repository_id)
                checksum_r.append(repo.checksum(include_dependencies = True))

            c_hash = "%s|%s|v2" % (
                ",".join(sorted_r),
                ",".join(checksum_r),
            )
            sha = hashlib.sha1(const_convert_to_rawstring(c_hash))

            cache_key = "%s/%s" % (
                self._cache_prefix("injected_library_dependencies_test"),
                sha.hexdigest())
            cached = self._cacher.pop(cache_key)
            missing_map = cached

        if missing_map is None:
            # this is the system-wide blacklist
            blacklisted_deps = \
                self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['dep_blacklist']
            blacklist = set()

            injected = []
            for repository_id in repository_ids:
                repo = self.open_repository(repository_id)
                repo_injected = repo.listAllInjectedPackageIds()
                injected.extend(((x, repository_id) for x in repo_injected))

                # this is the repository-specific blacklist. pack everything
                # together
                blacklist.update(self._get_missing_dependencies_blacklist(
                        repository_id))

            # reorder package matches per repository to speed up the
            # generation of the per-package blacklist.
            pkg_map = {}
            for pkg_id, pkg_repo in injected:
                obj = pkg_map.setdefault(pkg_repo, [])
                obj.append(pkg_id)

            # this is per-package dependency blacklist support
            per_package_blacklist = set()
            for repository_id, package_ids in pkg_map.items():
                repo = self.open_repository(repository_id)

                for pkg_dep, bl_pkg_deps in blacklisted_deps.items():
                    pkg_ids, rc = repo.atomMatch(pkg_dep, multiMatch = True)
                    for package_id in package_ids:
                        if package_id in pkg_ids:
                            per_package_blacklist.update(bl_pkg_deps)
                            break

            # merge with the rest of the metadata
            blacklist |= per_package_blacklist

            qa = self.QA()
            missing_map = qa.test_missing_dependencies(
                self, injected, blacklist = blacklist)

        if missing_map:
            self.output(
                darkred(_("There are broken injected packages. Please fix.")),
                importance = 2,
                level = "error",
                header = red(" @@ "))
        else:
            self.output(
                blue(_("Injected packages are healthy. You lucky...")),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )

        if use_cache and cached is None and cache_key is not None:
            self._cacher.push(cache_key, missing_map)

        return set(missing_map.keys())

    def removed_reverse_dependencies_test(self, repository_ids,
                                          use_cache = True):
        """
        Test repositories against packages that have been removed
        from system (but not stored in an injected state) and are
        still in the repositories. For each of them, print a list
        of direct reverse dependencies that should be fixed.

        @param repository_ids: repository identifiers to consider
        @type repository_ids: list
        @keyword use_cache: use on-disk cache
        @type use_cache: bool
        @return: an ordered list of orphaned package matches
            (sorted by atom).
        @rtype: list
        """
        all_repositories = self.repositories()
        spm = self.Spm()

        result = None
        cached = None
        cache_key = None
        # check if result is cached
        if use_cache:

            checksum_r = []
            checksum_r_all = []
            repo_ck = {}

            sorted_r = sorted(repository_ids)
            for repository_id in sorted_r:
                repo = self.open_repository(repository_id)
                ck = repo.checksum(include_dependencies = True)
                repo_ck[repository_id] = ck
                checksum_r.append(ck)

            sorted_r_all = sorted(all_repositories)
            for repository_id in sorted_r_all:
                ck = repo_ck.get(repository_id)
                if ck is None:
                    repo = self.open_repository(repository_id)
                    ck = repo.checksum(include_dependencies = True)
                checksum_r_all.append(ck)

            c_hash = "%s|%s~%s|%s|%f|v3" % (
                ",".join(sorted_r),
                ",".join(checksum_r),
                ",".join(sorted_r_all),
                ",".join(checksum_r_all),
                spm.installed_mtime(),
            )
            sha = hashlib.sha1(const_convert_to_rawstring(c_hash))

            cache_key = "%s/%s" % (
                self._cache_prefix("removed_reverse_dependencies_test"),
                sha.hexdigest())
            cached = self._cacher.pop(cache_key)
            result = cached

        if result is None:
            _ignore, removed, _ignore = self.scan_package_changes(
                repository_ids = all_repositories,
                removal_repository_ids = all_repositories)

            rsort = lambda x: self.open_repository(
                x[1]).retrieveAtom(x[0])
            rfilter = lambda x: x[1] in repository_ids

            # do not consider the repository if it's not
            # in the list.
            r_matches = list(filter(rfilter, removed))
            r_matches.sort(key = rsort)

            result = []
            for package_id, repository_id in r_matches:

                repo = self.open_repository(repository_id)
                reverse_package_ids = repo.retrieveReverseDependencies(
                    package_id)

                # filter out packages pointing to multiple slots
                sure_reverse_package_ids = set()
                for pkg_id in reverse_package_ids:
                    pkg_deps_size = 0
                    for pkg_dep in repo.retrieveDependencies(pkg_id):
                        pkg_dep_ids, _rc = repo.atomMatch(
                            pkg_dep, multiMatch = True)
                        if package_id in pkg_dep_ids:
                            # found my dependency back
                            pkg_deps_slots = set(
                                [repo.retrieveSlot(x) for x in pkg_dep_ids])
                            pkg_deps_size = max(
                                pkg_deps_size, len(pkg_deps_slots)
                            )

                    if pkg_deps_size == 1:
                        # if there is only one slot, then it's likely that the
                        # offending package will get its dependencies broken
                        # if removed.
                        sure_reverse_package_ids.add(pkg_id)

                result.append(
                    (package_id, repository_id, sure_reverse_package_ids))

            result = tuple(result)  # for caching

        if result:
            self.output(
                "[%s] %s:" % (
                    red(_("test")),
                    blue(_("these packages haven't been removed yet")),
                ),
                importance = 1,
                level = "warning",
                header = purple(" @@ ")
            )

        # generate atom_set for output purposes
        atom_set = set()
        for package_id, repository_id, _ignore in result:
            repo = self.open_repository(repository_id)
            r_atom = repo.retrieveAtom(package_id)
            atom_set.add(r_atom)

        for package_id, repository_id, rev_deps in result:

            repo = self.open_repository(repository_id)
            r_atom = repo.retrieveAtom(package_id)

            self.output(
                "[%s] %s" % (
                    brown(repository_id),
                    teal(r_atom),
                ),
                importance = 1,
                level = "warning",
                header = purple("  # ")
            )

            if rev_deps:
                self.output(
                    "%s:" %(
                        red(_("Needed by")),
                    ),
                    level = "warning",
                    header = purple("    # ")
                )

            for rev_pkg_id in rev_deps:
                rev_atom = repo.retrieveAtom(rev_pkg_id)
                if rev_atom is None:
                    rev_atom = _("corrupted entry")

                if rev_atom in atom_set:
                    atom_str = "%s (%s)" % (
                        brown(rev_atom),
                        darkgreen(_("removed")),)
                else:
                    atom_str = darkgreen(rev_atom)

                self.output(
                    atom_str,
                    level = "warning",
                    header = purple("    # ")
                )

        if use_cache and cached is None and cache_key is not None:
            self._cacher.push(cache_key, result)

        return [(x, y) for x, y, _z in result]

    def revision_downgrades_test(self, repository_ids,
                                 use_cache = True):
        """
        Test against revision downgrades, we assume that if a package
        in a repository x has a lesser revision than the same package in
        repository x - 1, it might be a source of troubles. package equality
        is when atom_1 = atom_2 for each couples of elements in x and x - 1.
        There are O(nm) couples.

        @param repository_ids: repository identifiers to consider
        @type repository_ids: list
        @keyword use_cache: use on-disk cache
        @type use_cache: bool
        @return: an ordered list of package matches with problems
            (sorted by atom).
        @rtype: list
        """
        result = None
        cached = None
        cache_key = None
        # check if result is cached
        if use_cache:

            checksum_r = []
            repo_ck = {}

            sorted_r = sorted(repository_ids)
            for repository_id in sorted_r:
                repo = self.open_repository(repository_id)
                ck = repo.checksum()
                repo_ck[repository_id] = ck
                checksum_r.append(ck)

            c_hash = "%s|%s|v1" % (
                ",".join(sorted_r),
                ",".join(checksum_r),
            )
            sha = hashlib.sha1(const_convert_to_rawstring(c_hash))

            cache_key = "%s/%s" % (
                self._cache_prefix("revision_downgrades_test"),
                sha.hexdigest())
            cached = self._cacher.pop(cache_key)
            result = cached

        if result is None:
            atom_map = {}
            faulty_matches = []

            for repository_id in reversed(repository_ids):
                repo = self.open_repository(repository_id)

                for package_id in repo.listAllPackageIds():
                    atom = repo.retrieveAtom(package_id)

                    obj = atom_map.setdefault(atom, collections.deque())
                    current_rev = repo.retrieveRevision(package_id)

                    for o_pkg_id, o_repository_id, o_revision in obj:
                        if current_rev < o_revision:
                            faulty_matches.append(
                                (package_id, repository_id, atom))
                            break

                    obj.append((package_id, repository_id, current_rev))

            # order by atom
            faulty_matches.sort(key = lambda x: x[2])
            result = tuple(faulty_matches)

        if result:
            self.output(
                "[%s] %s:" % (
                    red(_("test")),
                    blue(_("these packages have lower than"
                           " expected revisions")),
                ),
                importance = 1,
                level = "warning",
                header = purple(" @@ ")
            )

        for package_id, repository_id, atom in result:

            self.output(
                "[%s] %s" % (
                    brown(repository_id),
                    teal(atom),
                ),
                importance = 1,
                level = "warning",
                header = purple("  # ")
            )

        if use_cache and cached is None and cache_key is not None:
            self._cacher.push(cache_key, result)

        return [(x, y) for x, y, _z in result]

    def extended_dependencies_test(self, repository_ids, use_cache = True):
        """
        Test repository against missing dependencies.
        Moreover, the base repository (the first listed in server.conf)
        is tested by itself, since it must always be self-contained.

        @param repository_ids: list of repository identifiers to test
        @type repository_ids: list
        @keyword use_cache: use on-disk cache
        @type use_cache: bool
        @return: list (set) of unsatisfied dependencies
        @rtype: set
        """
        # just run this to make dev aware
        unsatisfied_deps = set()
        for repository_id in repository_ids:
            unsatisfied_deps |= self.dependencies_test(
                repository_id, use_cache = use_cache)

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        base_repository_id = srv_set['base_repository_id']
        if base_repository_id is not None:
            # dependency test base repository, which must always be
            # self-contained
            unsatisfied_deps |= self.dependencies_test(
                base_repository_id,
                match_repo = [base_repository_id],
                use_cache = use_cache)

        # given that we test all the repos, at least pick those where
        # QA is allowed
        all_repositories = self.qa_repositories()
        # test draining and merging as well, since we don't want
        # to get surprises when stuff is moved.
        unsatisfied_deps |= self.drained_dependencies_test(
            all_repositories, use_cache = use_cache)
        # check whethere there are packages no longer installed
        # that are still dependencies of other packages.
        self.removed_reverse_dependencies_test(
            all_repositories, use_cache = use_cache)

        # test library-level linking for injected packages as well.
        injected_matches = self.injected_library_dependencies_test(
            all_repositories, use_cache = use_cache)
        for package_id, repository_id in injected_matches:
            repo = self.open_repository(repository_id)
            unsatisfied_deps.add(repo.retrieveAtom(package_id))

        # test against revision downgrades
        self.revision_downgrades_test(all_repositories, use_cache = use_cache)

        return unsatisfied_deps

    def dependencies_test(self, repository_id, match_repo = None,
                          use_cache = True):
        """
        Test repository against missing dependencies.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword match_repo: list of repositories to look for missing deps
        @type match_repo: list
        @keyword use_cache: use on-disk cache
        @type use_cache: bool
        @return: list (set) of unsatisfied dependencies
        @rtype: set
        """
        mytxt = "%s %s" % (blue(_("Running dependencies test")), red("..."))
        self.output(
            mytxt,
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )

        deps_not_matched = None
        cached = None
        cache_key = None
        if use_cache:
            repo = self.open_repository(repository_id)
            checksum_r = repo.checksum(include_dependencies = True)

            match_repo_s = "None"
            checksum_m = ["~"]
            if match_repo is not None:
                match_repo_s = ",".join(match_repo)
                del checksum_m[:]
                for repository_id in match_repo:
                    repo = self.open_repository(repository_id)
                    checksum_m.append(
                        repo.checksum(include_dependencies = True))

            c_hash = "%s|%s~%s|%s|v2" % (
                repository_id,
                checksum_r,
                match_repo_s,
                ",".join(checksum_m),
            )
            sha = hashlib.sha1(const_convert_to_rawstring(c_hash))

            cache_key = "%s/%s" % (
                self._cache_prefix("dependencies_test"),
                sha.hexdigest())
            cached = self._cacher.pop(cache_key)
            deps_not_matched = cached

        if deps_not_matched is None:
            deps_not_matched = self._deps_tester(
                repository_id, match_repo = match_repo)

        if deps_not_matched:
            repository_ids = self.repositories()
            crying_atoms = {}

            for atom in deps_not_matched:
                for repository_id in repository_ids:
                    repo = self.open_repository(repository_id)
                    riddep = repo.searchDependency(atom)
                    if riddep == -1:
                        continue
                    rpackage_ids = repo.searchPackageIdFromDependencyId(riddep)
                    for i in rpackage_ids:
                        iatom = repo.retrieveAtom(i)
                        if atom not in crying_atoms:
                            crying_atoms[atom] = set()
                        crying_atoms[atom].add((iatom, repository_id))

            mytxt = blue("%s:") % (_("These are the dependencies not found"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            mytxt = "%s:" % (_("Needed by"),)
            for atom in deps_not_matched:
                self.output(
                    red(atom),
                    importance = 1,
                    level = "info",
                    header = blue("   # ")
                )
                if atom in crying_atoms:
                    self.output(
                        red(mytxt),
                        importance = 0,
                        level = "info",
                        header = blue("      # ")
                    )
                    for my_dep, myrepo in crying_atoms[atom]:
                        self.output(
                            "[%s:%s] %s" % (
                                blue(_("by repo")),
                                darkred(myrepo),
                                darkgreen(my_dep),
                            ),
                            importance = 0,
                            level = "info",
                            header = blue("      # ")
                        )
        else:
            mytxt = blue(_("Every dependency is satisfied. It's all fine."))
            self.output(
                mytxt,
                importance = 2,
                level = "info",
                header = red(" @@ ")
            )

        if use_cache and cached is None and cache_key is not None:
            self._cacher.push(cache_key, deps_not_matched)

        return deps_not_matched

    def test_shared_objects(self, repository_id, dump_results_to_file = False):
        """
        Test packages in repository, against missing shared objects.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword dump_results_to_file: dump results to file
        @type dump_results_to_file: bool
        @return: execution status (0 = fine)
        @rtype: int
        """
        pkg_list_path = None
        if dump_results_to_file:
            tmp_dir = const_mkdtemp(prefix="entropy.server")
            pkg_list_path = os.path.join(tmp_dir, "libtest_broken.txt")
            dmp_data = [
                (_("Broken and matched packages list"), pkg_list_path,),
            ]
            mytxt = "%s:" % (purple(_("Dumping results into these files")),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = blue(" @@ ")
            )
            for txt, path in dmp_data:
                mytxt = "%s: %s" % (blue(txt), path,)
                self.output(
                    mytxt,
                    importance = 0,
                    level = "info",
                    header = darkgreen("   ## ")
                )

        # load db
        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)
        QA = self.QA()
        packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn,
            broken_symbols = True, dump_results_to_file = dump_results_to_file)
        if status != 0:
            return 1

        if (not brokenexecs) and (not packages_matched):
            mytxt = "%s." % (_("System is healthy"),)
            self.output(
                blue(mytxt),
                importance = 2,
                level = "info",
                header = red(" @@ ")
            )
            return 0

        mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
        self.output(
            blue(mytxt),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        real_brokenexecs = [os.path.realpath(x) for x in brokenexecs if \
            x != os.path.realpath(x)]
        brokenexecs.update(real_brokenexecs)
        packages = self.Spm().search_paths_owners(brokenexecs)

        if packages:
            mytxt = "%s:" % (_("These are the matched packages"),)
            self.output(
                red(mytxt),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            for my_atom, my_elf_id in packages:
                package_slot = my_atom, my_elf_id
                self.output(
                    "%s [elf:%s]" % (
                        purple(my_atom),
                        darkgreen(str(my_elf_id)),
                    ),
                    importance = 0,
                    level = "info",
                    header = red("     # ")
                )
                for filename in sorted(packages[package_slot]):
                    self.output(
                        darkgreen(filename),
                        importance = 0,
                        level = "info",
                        header = brown("       => ")
                    )

            pkgstring_list = sorted(["%s%s%s" % (
                entropy.dep.dep_getkey(x[0]), etpConst['entropyslotprefix'],
                    x[1],) for x in sorted(packages)])
            if pkg_list_path is not None:
                enc = etpConst['conf_encoding']
                with codecs.open(pkg_list_path, "w", encoding=enc) as pkg_f:
                    for pkgstr in pkgstring_list:
                        pkg_f.write(pkgstr + "\n")

            pkgstring = ' '.join(pkgstring_list)
            mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
        else:
            self.output(
                red(_("No matched packages")),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )

        return 0

    def close_repositories(self, mask_clear = False):

        srv_dbcache = self._server_dbcache
        if srv_dbcache is not None:
            for item in srv_dbcache.keys():
                try:
                    srv_dbcache[item].close()
                except ProgrammingError: # already closed?
                    pass
            srv_dbcache.clear()
        if mask_clear:
            self._settings.clear()

    def commit_repositories(self):
        """
        Execute commit on all the open (in rw) repositories.
        """
        srv_dbcache = self._server_dbcache
        if srv_dbcache is not None:
            for repo in srv_dbcache.values():
                # commit on readonly repos has no effect
                repo.commit()

    def close_repository(self, entropy_repository):
        """
        Close single EntropyRepositoryBase instance, given its class object.

        @param entropy_repository: EntropyRepositoryBase instance
        @type entropy_repository: entropy.db.skel.EntropyRepositoryBase
        """
        found = None
        for item in self._server_dbcache:
            if entropy_repository is self._server_dbcache[item]:
                found = item
                break
        if found is not None:
            instance = self._server_dbcache.pop(found)
            instance.close()

    def repository_metadata(self, repository_id):
        """
        Return Entropy Server repository configuration metadata.

        @return: repository configuration metadata
        @rtype: dict
        @raise KeyError: if repository is not configured or available
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]

    def available_repositories(self):
        """
        Return a dictionary representing available repositories metadata.

        @return: available repository metadata
        @rtype: dict
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'].copy()

    def switch_default_repository(self, repository_id, save = None,
        handle_uninitialized = True):
        """
        Change Entropy Server default (current) repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword save: if True, save the default repository in server
            configuration
        @type save: bool
        @keyword handle_uninitialized: if repository is uninitialized, handle
            the case automatically
        @type handle_uninitialized: bool
        """
        # avoid setting __default__ as default server repo
        if repository_id == InstalledPackagesRepository.NAME:
            return
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']

        if save is None:
            save = self._save_repository
        if repository_id not in srv_set['repositories']:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                        repository_id,
                        _("repository not configured"),
                    )
            )
        self.close_repositories()
        srv_set['default_repository_id'] = repository_id
        self._repository = repository_id
        self._setup_services()
        if save:
            self._save_default_repository(repository_id)

        self._setup_community_repositories_settings()
        if handle_uninitialized:
            self._handle_uninitialized_repository(repository_id)

    def _setup_community_repositories_settings(self):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if srv_set['community_mode']:
            for repoid in srv_set['repositories']:
                srv_set['repositories'][repoid]['community'] = True

    def _handle_uninitialized_repository(self, repoid):

        if self._is_repository_initialized(repoid):
            return

        mytxt = blue("%s.") % (
            _("Your default repository is not initialized"),)
        self.output(
            "[%s:%s] %s" % (
                brown("repo"),
                purple(repoid),
                mytxt,
            ),
            importance = 1,
            level = "warning",
            header = darkred(" !!! ")
        )
        answer = self.ask_question(
            _("Do you want to initialize your default repository ?"))
        if answer == _("No"):
            mytxt = red("%s.") % (
                _("Continuing with an uninitialized repository"),)
            self.output(
                "[%s:%s] %s" % (
                    brown("repo"),
                    purple(repoid),
                    mytxt,
                ),
                importance = 1,
                level = "warning",
                header = darkred(" !!! ")
            )
        else:
            # move empty database for security sake
            dbfile = self._get_local_repository_file(repoid)
            if os.path.isfile(dbfile):
                shutil.move(dbfile, dbfile+".backup")
            self.initialize_repository(repoid)

    def _save_default_repository(self, repoid):

        # avoid setting __system__ as default server repo
        if repoid == InstalledPackagesRepository.NAME:
            return

        enc = etpConst['conf_encoding']
        server_conf = ServerSystemSettingsPlugin.server_conf_path()
        try:
            with codecs.open(server_conf, "r", encoding=enc) as f_srv:
                content = f_srv.readlines()
        except IOError as err:
            if err.errno == errno.ENOENT:
                content = None
            else:
                raise

        if content is not None:
            content = [x.strip() for x in content]
            found = False
            new_content = []
            for line in content:
                key, value = entropy.tools.extract_setting(line)
                if key == "default-repository":
                    line = "default-repository = %s" % (repoid,)
                    found = True
                new_content.append(line)
            if not found:
                new_content.append("default-repository = %s" % (repoid,))

            tmp_fd, tmp_path = const_mkstemp(prefix="_save_default_repository")
            with entropy.tools.codecs_fdopen(tmp_fd, "w", enc) as f_srv_t:
                for line in new_content:
                    f_srv_t.write(line+"\n")

            entropy.tools.rename_keep_permissions(
                tmp_path, server_conf)

        else:
            with codecs.open(server_conf, "w", encoding=enc) as f_srv:
                f_srv.write("default-repository = %s\n" % (repoid,))

    def enable_repository(self, repository_id):
        """
        Enable a repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @param enable: True for enable, False for disable
        @type enable: bool
        @return: True, if switch went fine, False otherwise
        @rtype: bool
        """
        return self._toggle_repository(repository_id, True)

    def disable_repository(self, repository_id):
        """
        Disable a repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: True, if switch went fine, False otherwise
        @rtype: bool
        """
        return self._toggle_repository(repository_id, False)

    def _toggle_repository(self, repository_id, enable):
        """
        Enable or disable a repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @param enable: True for enable, False for disable
        @type enable: bool
        @return: True, if switch went fine, False otherwise
        @rtype: bool
        """

        # avoid setting __default__ as default server repo
        if repository_id == InstalledPackagesRepository.NAME:
            return False

        server_conf = ServerSystemSettingsPlugin.server_conf_path()
        enc = etpConst['conf_encoding']
        try:
            with codecs.open(server_conf, "r", encoding=enc) as f_srv:
                content = [x.strip() for x in f_srv.readlines()]
        except IOError as err:
            if err.errno != errno.ENOENT:
                raise
            return None

        tmp_fd, tmp_path = const_mkstemp(prefix="_toggle_repository")
        with entropy.tools.codecs_fdopen(tmp_fd, "w", enc) as f_tmp:
            status = False
            for line in content:
                key, value = entropy.tools.extract_setting(line)
                if key is not None:
                    key = key.replace(" ", "")
                    key = key.replace("\t", "")
                    if key in ("repository", "#repository"):
                        if enable and (key == "#repository"):
                            line = "repository = %s" % (value,)
                            status = True
                        elif not enable and (key == "repository"):
                            line = "# repository = %s" % (value,)
                            status = True
                f_tmp.write(line+"\n")

        if status:
            entropy.tools.rename_keep_permissions(
                tmp_path, server_conf)
            self.close_repositories()
            self._settings.clear()
            self._setup_services()
        return status

    def _is_repository_initialized(self, repo):

        def do_validate(dbc):
            try:
                dbc.validate()
                return True
            except SystemDatabaseError:
                return False

        try:
            dbc = self.open_server_repository(repo, just_reading = True)
        except RepositoryError:
            return False
        valid = do_validate(dbc)
        if not valid:
            dbc = self.open_server_repository(repo, read_only = False,
                no_upload = True, is_new = True)
            valid = do_validate(dbc)

        return valid

    def _server_repository_sync_lock(self, repo, no_upload):

        if repo is None:
            repo = self._repository

        # check if the database is locked locally
        lock_file = self._get_repository_lockfile(repo)
        if os.path.isfile(lock_file):
            self.output(
                red(_("Entropy repository is already locked by you :-)")),
                importance = 1,
                level = "info",
                header = red(" * ")
            )
        else:
            # check if the database is locked REMOTELY
            mytxt = "%s ..." % (_("Locking and Syncing Entropy repository"),)
            self.output(
                red(mytxt),
                importance = 1,
                level = "info",
                header = red(" * "),
                back = True
            )
            for uri in self.remote_repository_mirrors(repo):

                crippled_uri = EntropyTransceiver.get_uri_name(uri)

                given_up = self.Mirrors.mirror_locked(repo, uri)
                if given_up:
                    mytxt = "%s:" % (_("Mirrors status table"),)
                    self.output(
                        darkgreen(mytxt),
                        importance = 1,
                        level = "info",
                        header = brown(" * ")
                    )
                    dbstatus = self.Mirrors.mirrors_status(repo)
                    for db_uri, db_st1, db_st2 in dbstatus:
                        db_st1_info = darkgreen(_("Unlocked"))
                        if db_st1:
                            db_st1_info = red(_("Locked"))
                        db_st2_info = darkgreen(_("Unlocked"))
                        if db_st2:
                            db_st2_info = red(_("Locked"))

                        crippled_uri = EntropyTransceiver.get_uri_name(db_uri)
                        self.output(
                            "%s: [%s: %s] [%s: %s]" % (
                                bold(crippled_uri),
                                brown(_("repository")),
                                db_st1_info,
                                brown(_("download")),
                                db_st2_info,
                            ),
                            importance = 1,
                            level = "info",
                            header = "\t"
                        )

                    raise OnlineMirrorError("OnlineMirrorError: %s %s" % (
                            _("cannot lock mirror"),
                            crippled_uri,
                        )
                    )

            # if we arrive here, it is because all the mirrors are unlocked
            self.Mirrors.lock_mirrors(repo, True)
            self.Mirrors.sync_repository(repo, enable_upload = not no_upload)


    def _init_generic_memory_server_repository(self, repository_id, description,
        pkg_mirrors = None, repo_mirrors = None, community_repo = False,
        service_url = None, set_as_default = False):

        if pkg_mirrors is None:
            pkg_mirrors = []
        if repo_mirrors is None:
            repo_mirrors = []

        repo = self._open_temp_repository(repository_id, temp_file = ":memory:")
        self._memory_db_srv_instances[repository_id] = repo

        # add to settings
        repodata = ServerSystemSettingsPlugin._generate_repository_metadata(
            repository_id, description, repo_mirrors, pkg_mirrors, False)
        repodata.update({
            'community': community_repo,
            'handler': '', # not supported
            '__temporary__': True,
            })
        ServerSystemSettingsPlugin.extend_repository_metadata(
            self._settings, repository_id, repodata)

        # this will make the change permanent until remove_repository()
        ServerSystemSettingsPlugin.REPOSITORIES[repository_id] = repodata
        # this will add our repo to the metadata stuff
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        srv_set['repositories'][repository_id] = repodata
        if set_as_default:
            # this will make the change permanent
            etpConst['defaultserverrepositoryid'] = repository_id
            # this will affect the current settings
            srv_set['default_repository_id'] = repository_id
        # not calling SystemSettings.clear()
        # because it's not strictly needed, but match metadata
        # could be available if this method gets called after
        # init_singleton(). In that case, it's up the user to call
        # clear()

        etp_repo_meta = {
            'lock_remote': False,
            'no_upload': True,
            'output_interface': self,
            'read_only': False,
            'local_dbfile': None,
            '__temporary__': True,
        }
        srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta)
        repo.add_plugin(srv_plug)
        return repo

    def _open_temp_repository(self, repo, temp_file = None, initialize = True):
        """
        Open temporary ServerPackagesRepository interface.

        @keyword dbname: database name
        @type dbname: string
        @keyword output_interface: entropy.output.TextInterface based instance
        @type output_interface: entropy.output.TextInterface based instance
        """
        if temp_file is None:
            tmp_fd, temp_file = const_mkstemp(prefix = 'entropy.server')
            os.close(tmp_fd)

        conn = ServerPackagesRepository(
            readOnly = False,
            dbFile = temp_file,
            name = repo,
            xcache = False,
            indexing = False,
            skipChecks = True,
            temporary = True
        )
        if initialize:
            conn.initializeRepository()
        return conn

    @staticmethod
    def get_repository(repository_id):
        """
        Reimplemented from entropy.client.interfaces.client.Client class
        """
        if repository_id == InstalledPackagesRepository.NAME:
            return InstalledPackagesRepository
        return ServerPackagesRepository

    def open_repository(self, repository_id):
        """
        This method aims to improve class usability by providing an easier
        method to open Entropy Server-side repositories like it happens
        with Entropy Client-side ones. If you just want open a read-only
        repository, feel free to use this wrapping method.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: ServerPackagesRepository instance
        @rtype: entropy.server.interfaces.ServerPackagesRepository
        """
        return self.open_server_repository(repository_id,
            just_reading = True, do_treeupdates = False)

    def remove_repository(self, repository_id, disable = False):
        """
        Remove Repository from those available, server-side override.
        """
        try:
            repo_data = ServerSystemSettingsPlugin.REPOSITORIES
            del repo_data[repository_id]
        except KeyError:
            pass

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        try:
            del srv_set['repositories'][repository_id]
        except KeyError as err:
            pass

        return super(Server, self).remove_repository(
            repository_id, disable = disable)

    def open_server_repository(self, repository_id, read_only = True,
            no_upload = True, just_reading = False, indexing = True,
            warnings = True, do_cache = True, use_branch = None,
            lock_remote = True, is_new = False, do_treeupdates = True):
        """
        Open Server-side Entropy repository given its repository identifier.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword read_only: open repository in read-only
        @type read_only: bool
        @param no_upload: allow to force the upload of the repository if
            required
        @type no_upload: bool
        @keyword just_reading: open the repository just for the purpose of
            getting data from it, without any fancy action. This avoids
            triggering some forced behaviours. If just_reading is True,
            read_only is forced to True.
        @type just_reading: bool
        @keyword indexing: allow repository indexing
        @type indexing: bool
        @keyword warnings: show warnings to user if True
        @type warnings: bool
        @keyword do_cache: use cache to retrieve a fresh repository object
        @type do_cache: bool
        @keyword use_branch: override default package branch when opening a
            new repository
        @type use_branch: bool
        @keyword lock_remote: lock remote mirrors when opening
        @type lock_remote: bool
        @keyword is_new: set this to True if the repository has been just
            created
        @type is_new: bool
        @type do_treeupdates: allow the execution of package names and slot
            updates
        @type do_treeupdates: bool
        @raise RepositoryError: if repository doesn't exist
        @return: EntropyRepositoryBase instance
        @rtype: EntropyRepositoryBase
        """
        # in-memory server repos
        if repository_id in self._memory_db_srv_instances:
            return self._memory_db_srv_instances[repository_id]

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repository_id == InstalledPackagesRepository.NAME and \
                srv_set['community_mode']:
            return self.installed_repository()

        if just_reading:
            read_only = True
            no_upload = True

        try:
            local_dbfile = self._get_local_repository_file(
                repository_id, branch = use_branch)
        except KeyError:
            # repository not available
            raise RepositoryError(repository_id)
        if do_cache:
            cached = self._server_dbcache.get(
                (repository_id, etpConst['systemroot'], local_dbfile, read_only,
                    no_upload, just_reading, use_branch, lock_remote,)
            )
            if cached != None:
                return cached

        local_dbfile_dir = os.path.dirname(local_dbfile)
        try:
            self._ensure_dir_path(local_dbfile_dir)
        except OSError as err:
            if err.errno != errno.EACCES:
                raise
            raise RepositoryError(repository_id)

        if (not read_only) and (lock_remote) and \
            (repository_id not in self._sync_lock_cache):
            self._server_repository_sync_lock(repository_id, no_upload)
            self._sync_lock_cache.add(repository_id)

        conn = ServerPackagesRepository(
            readOnly = read_only,
            dbFile = local_dbfile,
            name = repository_id,
            xcache = False # always set to False, if you want to enable
            # you need to make sure that client-side and server-side caches
            # don't collide due to sharing ServerPackagesRepository.repository_id()
        )
        etp_repo_meta = {
            'lock_remote': lock_remote,
            'no_upload': no_upload,
            'output_interface': self,
            'read_only': read_only,
            'local_dbfile': local_dbfile,
        }
        srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta)
        conn.add_plugin(srv_plug)

        valid = True
        try:
            conn.validate()
        except SystemDatabaseError:
            valid = False

        # verify if we need to update the database to sync
        # with portage updates, we just ignore being readonly in the case
        if (repository_id not in self._treeupdates_repos) and \
            (not just_reading):
            # sometimes, when filling a new server db
            # we need to avoid tree updates
            if valid:
                if self._inhibit_treeupdates:
                    # treeupdates are disabled
                    pass
                elif not do_treeupdates:
                    # requested not to run treeupdates
                    pass
                elif conn.readonly():
                    # repository is readonly, no shit
                    # readonly() always returns the effective
                    # write access to repository (despire what is
                    # really set during instantiation)
                    pass
                else:
                    self._repository_packages_spm_sync(repository_id, conn,
                        branch = use_branch)
            elif warnings and not is_new:
                mytxt = _("Repository is corrupted!")
                self.output(
                    darkred(mytxt),
                    importance = 1,
                    level = "warning",
                    header = bold(" !!! ")
                )

        if not read_only and valid and indexing:

            self.output(
                "[%s|%s] %s" % (
                        blue(repository_id),
                        red(_("repository")),
                        blue(_("indexing repository")),
                    ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            conn.createAllIndexes()

        if do_cache:
            # !!! also cache just_reading otherwise there will be
            # real issues if the connection is opened several times
            self._server_dbcache[
                (repository_id, etpConst['systemroot'], local_dbfile, read_only,
                no_upload, just_reading, use_branch, lock_remote,)] = conn

        # this ensures that side effect outcomes are committed
        # to db (treeupdates for example)
        conn.commit(force = True, no_plugins = True)
        return conn

    def _repository_packages_spm_sync(self, repository_id, repo_db,
        branch = None):
        """
        Service method used to sync package names with Source Package Manager.
        Source Package Manager can change package names, categories or slot
        and Entropy repositories must be kept in sync.
        """
        if branch is None:
            branch = self._settings['repositories']['branch']
        self._treeupdates_repos.add(repository_id)
        self.Spm().package_names_update(repo_db, repository_id, self, branch)

    def _setup_empty_repository(self, repository_path):

        dbdir = os.path.dirname(repository_path)
        if not os.path.isdir(dbdir):
            os.makedirs(dbdir)

        mytxt = red("%s ...") % (_("Initializing an empty repository"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" * "),
            back = True
        )
        dbconn = self.open_generic_repository(repository_path)
        dbconn.initializeRepository()
        dbconn.commit()
        dbconn.close()
        mytxt = "%s %s %s." % (
            red(_("Entropy repository file")),
            bold(repository_path),
            red(_("successfully initialized")),
        )
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

    def _get_whitelisted_licenses(self, repository_id):
        wl_file = self._get_local_repository_licensewhitelist_file(
            repository_id)
        if not os.path.isfile(wl_file):
            return []
        return entropy.tools.generic_file_content_parser(
            wl_file, encoding = etpConst['conf_encoding'])

    def _get_restricted_packages(self, repository_id):
        rl_file = self._get_local_restricted_file(repository_id)
        if not os.path.isfile(rl_file):
            return []
        return entropy.tools.generic_file_content_parser(rl_file,
            comment_tag = "##", encoding = etpConst['conf_encoding'])

    def _is_pkg_restricted(self, repository_id, pkg_atom, pkg_slot):

        restricted_pkgs = self._get_restricted_packages(repository_id)
        if not restricted_pkgs:
            return False

        pkg_key = entropy.dep.dep_getkey(pkg_atom)
        for r_dep in restricted_pkgs:
            r_key, r_slot = entropy.dep.dep_getkey(r_dep), \
                entropy.dep.dep_getslot(r_dep)
            if r_slot is None:
                r_slot = pkg_slot

            if (r_key == pkg_key) and (r_slot == pkg_slot):
                return True

        return False

    def _is_pkg_free(self, repository_id, pkg_licenses):

        # check if nonfree directory support is enabled, if not,
        # always return True.
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if not srv_set['nonfree_packages_dir_support']:
            return True

        wl_licenses = self._get_whitelisted_licenses(repository_id)

        if not pkg_licenses:
            return True # free if no licenses provided
        if not wl_licenses:
            return True # no whitelist !

        for lic in pkg_licenses:
            if lic not in wl_licenses:
                return False
        return True

    def _package_injector(self, repository_id, package_files, inject = False):

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']

        def _package_injector_check_license(pkg_data):
            licenses = pkg_data['license'].split()
            return self._is_pkg_free(repository_id, licenses)

        def _package_injector_check_restricted(pkg_data):
            pkgatom = entropy.dep.create_package_atom_string(
                pkg_data['category'], pkg_data['name'], pkg_data['version'],
                pkg_data['versiontag'])
            return self._is_pkg_restricted(repository_id,
                pkgatom, pkg_data['slot'])

        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        package_file = package_files[0]
        self.output(
            "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    _("adding package"),
                    bold(os.path.basename(package_file)),
                ),
            importance = 1,
            level = "info",
            header = brown(" * "),
            back = True
        )
        mydata = self.Spm().extract_package_metadata(package_file,
            license_callback = _package_injector_check_license,
            restricted_callback = _package_injector_check_restricted)
        is_licensed_ugly = not _package_injector_check_license(mydata)
        is_restricted = _package_injector_check_restricted(mydata)

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            # GPG not available
            repo_sec = None

        def _generate_extra_download(path, down_type):
            extra_url = entropy.tools.create_package_dirpath(mydata['branch'],
                nonfree = is_licensed_ugly, restricted = is_restricted) + \
                    "/" + os.path.basename(path)
            size = entropy.tools.get_file_size(path)
            disksize = entropy.tools.get_uncompressed_size(path)
            md5 = entropy.tools.md5sum(path)
            sha1 = entropy.tools.sha1(path)
            sha256 = entropy.tools.sha256(path)
            sha512 = entropy.tools.sha512(path)
            gpg = None
            if repo_sec is not None:
                gpg = self._get_gpg_signature(repo_sec, repository_id, path)
            edw = {
                'download': extra_url,
                'type': down_type,
                'size': size,
                'disksize': disksize,
                'md5': md5,
                'sha1': sha1,
                'sha256': sha256,
                'sha512': sha512,
                'gpg': gpg,
            }
            return edw

        # ~~~
        # Support for separate debug package files
        extra_files = package_files[1:]
        debuginfo_files = []
        for extra_filepath in extra_files:
            if extra_filepath.endswith(etpConst['packagesdebugext']):
                debuginfo_files.append(extra_filepath)
        for debuginfo_filepath in debuginfo_files:
            extra_files.remove(debuginfo_filepath)

        extra_download = []
        for debuginfo_filepath in debuginfo_files:
            extra_download.append(
                _generate_extra_download(debuginfo_filepath, "debug"))

        for extra_filepath in extra_files:
            extra_download.append(
                _generate_extra_download(extra_filepath, "data"))

        self._pump_extracted_package_metadata(mydata, repository_id,
            {
                'injected': inject,
                'original_repository': repository_id,
                'extra_download': extra_download,
            }
        )
        package_id = dbconn.handlePackage(mydata)
        revision = dbconn.retrieveRevision(package_id)
        # make sure that info have been written to disk
        dbconn.commit()

        myserver_repos = list(srv_set['repositories'].keys())

        ### since we are handling more repositories, we need to make sure
        ### that there are no packages in other repositories with same atom
        ### and greater revision
        rev_test_atom = mydata['atom']
        max_rev = -1
        for myrepo in myserver_repos:

            # not myself
            if myrepo == repository_id:
                continue

            mydbconn = self.open_server_repository(myrepo, read_only = True,
                no_upload = True)

            myrepo_package_ids = mydbconn.getPackageIds(rev_test_atom)
            for myrepo_package_id in myrepo_package_ids:
                myrev = mydbconn.retrieveRevision(myrepo_package_id)
                if myrev > max_rev:
                    max_rev = myrev

        if max_rev >= revision:
            max_rev += 1
            revision = max_rev
            mydata['revision'] = revision
            # update revision for pkg now
            dbconn.setRevision(package_id, revision)

        # make sure that info have been written to disk, again
        dbconn.commit()

        # set trashed counters
        trashing_counters = set()

        for myrepo in myserver_repos:
            mydbconn = self.open_server_repository(myrepo, read_only = True,
                no_upload = True)
            mylist = mydbconn.getPackagesToRemove(
                    mydata['name'],
                    mydata['category'],
                    mydata['slot'],
                    mydata['injected']
            )
            for myitem in mylist:
                trashing_counters.add(mydbconn.retrieveSpmUid(myitem))

        for mycounter in trashing_counters:
            dbconn.setTrashedUid(mycounter)

        # make sure that info have been written to disk, again
        dbconn.commit()

        atom = dbconn.retrieveAtom(package_id)
        self.output(
            "[%s] %s: %s %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("added package")),
                    darkgreen(atom),
                    blue(_("rev")), # as in revision
                    bold(str(revision)),
                ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        manual_deps = sorted(dbconn.retrieveManualDependencies(package_id,
            resolve_conditional_deps = False))
        if manual_deps:
            self.output(
                "[%s] %s: %s" % (
                        darkgreen(repository_id),
                        blue(_("manual dependencies for")),
                        darkgreen(atom),
                    ),
                importance = 1,
                level = "warning",
                header = darkgreen("   ## ")
            )
            for m_dep in manual_deps:
                self.output(
                    brown(m_dep),
                    importance = 1,
                    level = "warning",
                    header = darkred("    # ")
                )

        download_url = self._setup_repository_package_filename(dbconn,
            package_id)
        destination_path = self.complete_local_upload_package_path(
            download_url, repository_id)
        destination_dir = os.path.dirname(destination_path)
        self._ensure_dir_path(destination_dir)

        # since destination_path determines the final path name
        # make sure it corresponds to the one in package_files
        # but copy the object first (shallow)
        move_files = package_files[:]
        package_file = move_files[0]
        # rename to final name first
        final_path = os.path.join(os.path.dirname(package_file),
            os.path.basename(destination_path))
        if package_file != final_path:
            os.rename(package_file, final_path)
            move_files[0] = final_path

        # run in reverse order, this way, the base package file will
        # be moved at the end, resulting in improved reliability.
        destination_paths = []
        for package_file in reversed(move_files):
            final_path = os.path.join(destination_dir,
                os.path.basename(package_file))
            destination_paths.append(final_path)
            try:
                os.rename(package_file, final_path)
            except OSError as err:
                if err.errno != errno.EXDEV:
                    raise
                shutil.move(package_file, final_path)

        # make sure that info have been written to disk, again
        dbconn.commit()

        # reverse it again, since we wrote it reversed already
        destination_paths.reverse()
        return package_id, destination_paths

    def _setup_repository_package_filename(self, repo, package_id):
        """
        Setup a new repository file name using current package metadata.
        """
        category = repo.retrieveCategory(package_id)
        name = repo.retrieveName(package_id)
        version = repo.retrieveVersion(package_id)
        tag = repo.retrieveTag(package_id)
        revision = repo.retrieveRevision(package_id)
        signatures = repo.retrieveSignatures(package_id)
        sha1 = None
        if signatures:
            sha1, _ignore, _ignore, _ignore = signatures

        old_download_url = repo.retrieveDownloadURL(package_id)
        download_dir = os.path.dirname(old_download_url)

        package_name = entropy.dep.create_package_filename(
            category, name, version, tag, ext = etpConst['packagesext'],
            revision = revision, sha1 = sha1)

        download_url = os.path.join(download_dir, package_name)
        repo.setDownloadURL(package_id, download_url)
        repo.commit()

        return download_url

    def __user_filter_out_missing_deps(self, pkg_repo, entropy_repository,
        missing_map, ask):

        def _show_missing_deps(missing_deps):
            self.output(
                "[%s] %s:" % (
                    darkgreen(pkg_repo),
                    teal(_("these are the missing dependencies")),
                ),
                importance = 1,
                level = "info",
                header = purple(" @@ ")
            )
            for pkg_match, deps in missing_deps.items():
                pkg_id, repo = pkg_match
                atom = entropy_repository.retrieveAtom(pkg_id)
                slot = entropy_repository.retrieveSlot(pkg_id)
                self.output(
                    "%s:%s" % (
                        teal(atom),
                        purple(slot),
                    ),
                    level = "info",
                    header = " :: "
                )
                for dep in sorted(deps):
                    self.output(
                        brown(dep),
                        level = "info",
                        header = red("    # ")
                    )

        missing_deps = {}
        if not ask:
            # not interactive, add everything
            for pkg_match, missing_extended in missing_map.items():
                if not missing_extended:
                    continue
                obj = missing_deps.setdefault(pkg_match, set())
                for dep_list in missing_extended.values():
                    obj.update(dep_list)

            _show_missing_deps(missing_deps)
            return missing_deps

        header_txt = """\
# Some missing dependencies have been found.
# Please use this text file to commit the ones you want
# to get added to their respective packages.
#
# HOWTO: just remove the lines containing unwanted dependencies and
# keep those you think are sane.
#
# IMPORTANT:
# Lines starting with "#" are comments and will be ignored.
# Lines starting with "##" are reserved to this application for
# parsing purposes.


"""

        editor_lines = []
        for pkg_match, missing_extended in missing_map.items():

            if not missing_extended:
                # wtf
                continue

            pkg_id, missing_pkg_repo = pkg_match
            if pkg_repo != missing_pkg_repo:
                raise AttributeError("this cannot happen")

            atom = entropy_repository.retrieveAtom(pkg_id)
            slot = entropy_repository.retrieveSlot(pkg_id)
            line = "## %s %s => %s:%s" % (pkg_id, pkg_repo, atom, slot)
            editor_lines.append(const_convert_to_unicode(line))

            for lib_match, dep_list in missing_extended.items():
                library, elf = lib_match
                line = "# %s, %s" % (library, elf)
                editor_lines.append(const_convert_to_unicode(line))
                for dep in sorted(dep_list):
                    editor_lines.append(dep)
            editor_lines.append(const_convert_to_unicode(""))
            editor_lines.append(const_convert_to_unicode(""))

        if not editor_lines:
            # wtf!?
            return {}

        enc = etpConst['conf_encoding']
        tmp_path = None
        while True:

            if tmp_path is None:
                tmp_fd, tmp_path = const_mkstemp(prefix = 'entropy.server',
                    suffix = ".conf")
                with entropy.tools.codecs_fdopen(tmp_fd, "w", enc) as tmp_f:
                    tmp_f.write(header_txt)
                    for editor_line in editor_lines:
                        tmp_f.write(editor_line)
                        tmp_f.write("\n")

            success = self.edit_file(tmp_path)
            if not success:
                # retry ?
                os.remove(tmp_path)
                tmp_path = None
                continue

            # parse the file back, build missing_deps
            all_good = True
            missing_deps = {}
            with codecs.open(tmp_path, "r", encoding=enc) as tmp_f:
                pkg_match = None
                for line in tmp_f.readlines():
                    line = line.strip()

                    if not line:
                        # empty line
                        continue

                    elif line.startswith("##"):
                        # new package match, hopefully
                        line_data = line.lstrip("## ").split()
                        if len(line_data) < 2:
                            # something is really bad
                            all_good = False
                            break
                        try:
                            pkg_id = int(line_data[0])
                        except ValueError:
                            # something is really bad
                            all_good = False
                            break

                        repo = line_data[1]
                        if repo != pkg_repo:
                            # bad here too
                            all_good = False
                            break

                        pkg_match = pkg_id, repo
                        continue

                    elif line.startswith("#"):
                        # ignore comment line
                        continue

                    elif pkg_match:
                        # real line, should contain a valid dependency string
                        # if pkg_match is set
                        obj = missing_deps.setdefault(pkg_match, set())
                        obj.add(line)

            if not all_good:
                os.remove(tmp_path)
                tmp_path = None
                continue

            if not missing_deps:
                self.output(
                    "[%s] %s:" % (
                        darkgreen(pkg_repo),
                        teal(_("no missing dependencies !")),
                    ),
                    importance = 1,
                    level = "warning",
                    header = red(" @@ ")
                )
            else:
                _show_missing_deps(missing_deps)

            # ask confirmation
            while True:
                try:
                    rc_question = self.ask_question(
                        "[%s] %s" % (
                            purple(pkg_repo),
                            teal(_("Do you agree?"))
                        ),
                        responses = (_("Yes"), _("Repeat"),)
                    )
                except KeyboardInterrupt:
                    # do not allow, we're in a critical region
                    continue
                break
            if rc_question == _("Yes"):
                break
            # otherwise repeat everything again
            # keep tmp_path


        if tmp_path is not None:
            try:
                os.remove(tmp_path)
            except (OSError) as err:
                if err.errno != errno.ENOENT:
                    raise
        return missing_deps

    def missing_runtime_dependencies_test(self, package_matches, ask = True,
        bump_packages = False):
        """
        Use Entropy QA interface to check package matches against missing
        runtime dependencies, adding them.

        @param package_matches: list of Entropy package matches
        @type package_matches: list
        @keyword ask: if missing runtime dependencies should be validated
            interactively
        @type ask: bool
        @keyword bump_packages: True, if packages should be bumped (revision
            bumped)
        @type bump_packages: bool
        """
        blacklisted_deps = \
            self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['dep_blacklist']
        my_qa = self.QA()

        def get_blacklisted_deps(package_ids, repo_db):
            my_blacklist = set()
            for pkg_dep, bl_pkg_deps in blacklisted_deps.items():
                pkg_ids, rc = repo_db.atomMatch(pkg_dep, multiMatch = True)
                for package_id in package_ids:
                    if package_id in pkg_ids:
                        my_blacklist.update(bl_pkg_deps)
                        break
            return my_blacklist

        pkg_map = {}
        for pkg_id, pkg_repo in package_matches:
            obj = pkg_map.setdefault(pkg_repo, [])
            obj.append(pkg_id)

        for pkg_repo, package_ids in pkg_map.items():

            repo_blacklist = self._get_missing_dependencies_blacklist(pkg_repo)

            dbconn = self.open_server_repository(pkg_repo, read_only = False,
                no_upload = True)
            pkg_blacklisted_deps = get_blacklisted_deps(package_ids, dbconn)
            pkg_blacklisted_deps |= repo_blacklist

            # missing dependencies check
            target_matches = [(x, pkg_repo) for x in package_ids]
            missing_map = my_qa.test_missing_dependencies(
                self, target_matches, blacklist = pkg_blacklisted_deps)
            # also warn about potentially missing library deps
            _broken_matches = my_qa.warn_missing_dependencies(
                self, target_matches)

            missing_deps = self.__user_filter_out_missing_deps(pkg_repo,
                dbconn, missing_map, ask)

            for (pkg_id, missing_pkg_repo), missing in missing_deps.items():
                if pkg_repo != missing_pkg_repo:
                    # with current API, this never happens!
                    # but since this is a critical region, better being
                    # safe than sorry.
                    # pkg_repo is always the same...
                    raise AssertionError(
                        "pkg_repo and missing_pkg_repo must be equal")

                if bump_packages:
                    # in this case, a new package should be generated, with
                    # bumped revision
                    pkg_data = dbconn.getPackageData(pkg_id)
                    # also bump injected packages properly
                    original_injected = pkg_data['injected']
                    pkg_data['injected'] = False
                    pkg_id = dbconn.handlePackage(pkg_data)
                    if original_injected:
                        dbconn.setInjected(pkg_id)
                    # make sure that info have been written to disk
                    dbconn.commit()

                # NOTE that missing is a list here, so no fancy
                # dependency type is set.
                dbconn.insertDependencies(pkg_id, missing)

            if missing_deps:
                # save changes here again
                dbconn.commit()

    def _external_metadata_qa_hook_test(self, package_matches):
        """
        Execute external metadata QA check, if executable exists.
        Returns True for success, False for error. Warnings are not considered
        blocking.
        """
        qa_exec = os.path.join(SystemSettings.packages_config_directory(),
                               "packages.server.qa.exec")
        if not const_file_readable(qa_exec):
            return True

        # avoid privs escalation
        st = os.stat(qa_exec)
        file_uid = st[stat.ST_UID]
        file_gid = st[stat.ST_GID]
        if not ((file_uid == 0) and (file_gid == 0)):
            self.output(
                "[%s] %s: %s, %s" % (
                    purple("qa"),
                    brown(_("metadata QA hook")),
                    purple(qa_exec),
                    brown(_("not owned by uid and gid = 0")),
                ),
                importance = 1,
                level = "error",
                header = darkred(" !!! "),
            )
            return False

        count = 0
        maxcount = len(package_matches)
        self.output(
            "[%s] %s: %s" % (
                purple("qa"),
                teal(_("using metadata QA hook")),
                darkgreen(qa_exec),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ ")
        )
        qa_success = True
        for package_id, repository_id in package_matches:
            count += 1
            repo_db = self.open_server_repository(repository_id,
                read_only = False, no_upload = True)

            pkg_atom, pkg_name, pkg_version, pkg_tag, \
            pkg_description, pkg_category, pkg_chost, \
            pkg_cflags, pkg_cxxflags, pkg_homepage, \
            pkg_license, pkg_branch, pkg_uri, \
            pkg_md5, pkg_slot, pkg_etpapi, \
            pkg_date, pkg_size, pkg_rev = repo_db.getBaseData(package_id)
            pkg_deps = repo_db.retrieveDependenciesList(package_id)
            pkg_needed = repo_db.retrieveNeeded(package_id, extended = True)
            pkg_provided_libs = repo_db.retrieveProvidedLibraries(package_id)
            pkg_keywords = repo_db.retrieveKeywords(package_id)

            self.output(
                "[%s] %s: %s" % (
                    purple("qa"),
                    teal(_("metadata QA check for")),
                    darkgreen(pkg_atom),
                ),
                importance = 1,
                level = "info",
                header = blue(" @@ "),
                count = (count, maxcount),
                back = True
            )
            env = os.environ.copy()
            env['REPOSITORY_ID'] = str(repository_id)
            env['PKG_ID'] = str(package_id)
            env['PKG_ATOM'] = str(pkg_atom)
            env['PKG_NAME'] = str(pkg_name)
            env['PKG_VERSION'] = str(pkg_version)
            env['PKG_TAG'] = str(pkg_tag or "")
            env['PKG_DESCRIPTION'] = const_convert_to_rawstring(pkg_description)
            env['PKG_CATEGORY'] = str(pkg_category)
            env['PKG_CHOST'] = str(pkg_chost)
            env['PKG_CFLAGS'] = str(pkg_cflags)
            env['PKG_CXXFLAGS'] = str(pkg_cxxflags)
            env['PKG_HOMEPAGE'] = const_convert_to_rawstring(pkg_homepage)
            env['PKG_LICENSE'] = str(pkg_license)
            env['PKG_BRANCH'] = str(pkg_branch)
            env['PKG_KEYWORDS'] = str(" ".join(pkg_keywords))
            env['PKG_DOWNLOAD'] = str(pkg_uri)
            env['PKG_MD5'] = str(pkg_md5)
            env['PKG_SLOT'] = str(pkg_slot)
            env['PKG_ETPAPI'] = str(pkg_etpapi)
            env['PKG_DATE'] = str(pkg_date)
            env['PKG_SIZE'] = str(pkg_size)
            env['PKG_REVISION'] = str(pkg_rev)
            env['PKG_DEPS'] = str("\n".join(pkg_deps))
            env['PKG_NEEDED_LIBS'] = str("\n".join(["%s|%s" % (x, y) \
                    for x, y in pkg_needed]))
            env['PKG_PROVIDED_LIBS'] = str("\n".join(
                ["%s|%s|%s" % (x, y, z) for x, y, z in \
                    pkg_provided_libs]))

            # now call the script
            try:
                proc = subprocess.Popen(
                    [qa_exec], stdout = sys.stdout, stderr = sys.stderr,
                    stdin = sys.stdin, env = env)
                rc = proc.wait()
            except OSError as err:
                self.output(
                    "[%s] %s: %s, %s" % (
                        purple("qa"),
                        bold(_("cannot execute metadata QA hook for")),
                        darkgreen(pkg_atom),
                        repr(err),
                    ),
                    importance = 1,
                    level = "error",
                    header = darkred(" !!! "),
                    count = (count, maxcount)
                )
                rc = 2
                if const_debug_enabled():
                    import pdb
                    pdb.set_trace()

            if rc == 0:
                # all good
                continue
            elif rc == 1:
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                self.output(
                    "[%s] %s !" % (
                        darkred("qa"),
                        brown(_("attention, QA hook returned a warning")),
                    ),
                    importance = 1,
                    level = "warning",
                    header = darkred(" !!! "),
                    count = (count, maxcount)
                )
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
            else:
                # anything != 0 and 1 is considered error
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                self.output(
                    "[%s] %s !" % (
                        darkred("qa"),
                        brown(_("attention, QA hook returned an error")),
                    ),
                    importance = 1,
                    level = "error",
                    header = darkred(" !!! "),
                    count = (count, maxcount)
                )
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                qa_success = False

        self.output(
            "[%s] %s" % (
                purple("qa"),
                teal(_("metadata QA check complete")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ ")
        )

        return qa_success

    def _add_packages_qa_tests(self, package_matches, ask = True):
        """
        Execute some generic QA checks for broken libraries on packages added
        to repository.
        """
        self.missing_runtime_dependencies_test(package_matches, ask = ask)

        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        my_qa = self.QA()
        my_qa.test_missing_runtime_libraries(self, package_matches,
            base_repository_id = my_settings['base_repository_id'])

        qa_success = self._external_metadata_qa_hook_test(package_matches)
        if my_settings['broken_revdeps_qa_check']:
            my_qa.test_reverse_dependencies_linking(self, package_matches)
        return qa_success

    def add_packages_to_repository(self, repository_id, packages_data,
        ask = True):
        """
        Add package files to given repository. packages_data contains a list
        of tuples composed by (path to package files, execute_injection boolean).
        Injection is a way to avoid a package being removed from the repository
        automatically when an updated package is added.

        @param repository_id: repository identifier
        @type repository_id: string
        @param packages_data: list of tuples composed by ([path, path1], bool)
            The first path object represents the main package file.
            While the others represent further package files.
            If one ends with eptConst['packagesdebugext'], it will be considered
            as debuginfo package file.
        @type packages_data: list
        @return: list (set) of package identifiers added
        @rtype: set
        """
        mycount = 0
        maxcount = len(packages_data)
        package_ids_added = set()
        to_be_injected = set()

        for package_filepaths, inject in packages_data:

            mycount += 1
            for package_filepath in package_filepaths:
                header = blue(" @@ ")
                count = (mycount, maxcount,)
                if package_filepaths[0] != package_filepath:
                    self.output(
                        "%s" % (
                            brown(os.path.basename(package_filepath)),
                        ),
                        importance = 1,
                        level = "info",
                        header = teal("     # ")
                    )
                else:
                    self.output(
                        "[%s] %s: %s" % (
                            darkgreen(repository_id),
                            blue(_("adding package")),
                            darkgreen(os.path.basename(package_filepath)),
                        ),
                        importance = 1,
                        level = "info",
                        header = blue(" @@ "),
                        count = (mycount, maxcount,)
                    )

            if inject and len(package_filepaths) == 1:
                # just make sure user is aware of the fact that no separate
                # debug packages will be made.
                self.output(
                    "%s" % (
                        brown(_("injected package, no separate debug package")),
                    ),
                    importance = 1,
                    level = "info",
                    header = teal("     !! ")
                )

            try:
                # add to database
                package_id, destination_paths = self._package_injector(
                    repository_id, package_filepaths, inject = inject)
                package_ids_added.add(package_id)
                to_be_injected.add((package_id, destination_paths[0]))
            except Exception as err:
                entropy.tools.print_traceback()
                self.output(
                    "[%s] %s: %s" % (
                        darkgreen(repository_id),
                        darkred(_("Exception caught, closing tasks")),
                        darkgreen(str(err)),
                    ),
                    importance = 1,
                    level = "error",
                    header = bold(" !!! "),
                    count = (mycount, maxcount,)
                )
                # reinit librarypathsidpackage table
                if package_ids_added:
                    self._add_packages_qa_tests(
                        [(x, repository_id) for x in package_ids_added],
                        ask = ask)
                if to_be_injected:
                    self._inject_database_into_packages(repository_id,
                        to_be_injected)
                self.close_repositories()
                raise

        # make sure packages are really available, it can happen
        # after a previous failure to have garbage here
        dbconn = self.open_server_repository(repository_id, just_reading = True)
        package_ids_added = set((x for x in package_ids_added if \
            dbconn.isPackageIdAvailable(x)))

        if package_ids_added:
            self._add_packages_qa_tests(
                [(x, repository_id) for x in package_ids_added], ask = ask)

        # inject database into packages
        self._inject_database_into_packages(repository_id, to_be_injected)

        return package_ids_added

    def _taint_database(self, repository_id):

        # taint the database status
        db_file = self._get_local_repository_file(repository_id)
        taint_file = self._get_local_repository_taint_file(repository_id)
        with codecs.open(taint_file, "w") as f:
            f.write("repository tainted\n")

        const_setup_file(taint_file, etpConst['entropygid'], 0o664)
        ServerRepositoryStatus().set_tainted(db_file)

    def _bump_database(self, repository_id):
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        self._taint_database(repository_id)
        dbconn.commit()
        self.close_repository(dbconn)


    def _ensure_paths(self, repo):
        upload_dir = self._get_local_upload_directory(repo)
        db_dir = self._get_local_repository_dir(repo)
        for mydir in [upload_dir, db_dir]:
            if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)):
                os.makedirs(mydir, 0o775)
                const_setup_perms(mydir, etpConst['entropygid'],
                    recursion = False, uid = etpConst['uid'])

    def _ensure_dir_path(self, dir_path):
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path, 0o775)
            const_setup_perms(dir_path, etpConst['entropygid'],
                recursion = False, uid = etpConst['uid'])

    def _setup_services(self):
        self._setup_entropy_settings(self._repository)
        self._backup_entropy_settings()
        self.Mirrors = MirrorsServer(self, self._repository)

    def _setup_entropy_settings(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        backup_list = [
            'etpdatabaseclientfilepath',
            {'server': srv_set.copy()},
        ]
        for setting in backup_list:
            if setting not in self._settings_to_backup:
                self._settings_to_backup.append(setting)
        # setup entropy client repo
        if not srv_set['community_mode']:
            repo_data = srv_set['repositories'][repository_id]
            if "__temporary__" not in repo_data:
                etpConst['etpdatabaseclientfilepath'] = \
                    self._get_local_repository_file(repository_id)
        const_create_working_dirs()

    def _show_interface_status(self):
        type_txt = _("server-side repository")
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if srv_set['community_mode']:
            type_txt = _("community repository")
        # ..on repository: <repository_name>
        mytxt = _("Entropy Server Interface Instance on repository")
        self.output(
            "%s: %s" % (blue(mytxt),
                red(self._repository),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        self.output(
            "%s: %s (%s: %s)" % (
                brown(_("current branch")),
                darkgreen(self._settings['repositories']['branch']),
                purple(_("type")),
                bold(type_txt),
            ),
            importance = 1,
            level = "info",
            header = red("    ")
        )
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        repos = list(srv_set['repositories'].keys())
        mytxt = blue("%s:") % (_("Currently configured repositories"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )
        for repo in repos:
            self.output(
                darkgreen(repo),
                importance = 0,
                level = "info",
                header = brown("   # ")
            )

    def _backup_constant(self, constant_name):
        if constant_name not in etpConst:
            raise KeyError("%s not available" % (constant_name,))
        etpConst['backed_up'].update(
            {constant_name: copy.copy(etpConst[constant_name])})

    def _backup_entropy_settings(self):
        for setting in self._settings_to_backup:
            if isinstance(setting, const_get_stringtype()):
                self._backup_constant(setting)
            elif isinstance(setting, dict):
                self._settings.set_persistent_setting(setting)

    def _get_gpg_signature(self, repo_sec, repo, pkg_path):
        try:
            if not repo_sec.is_keypair_available(repo):
                return None # GPG is not enabled
        except RepositorySecurity.KeyExpired as err:
            self.output(
                "[%s] %s: %s, %s." % (
                    darkgreen(repo),
                    darkred(_("GPG key expired")),
                    err,
                    darkred(_("please frigging fix")),
                ),
                importance = 1,
                level = "warning",
                header = bold(" !!! ")
            )
            return None
        except RepositorySecurity.GPGError as err:
            self.output(
                "[%s] %s: %s, %s." % (
                    darkgreen(repo),
                    darkred(_("GPG got unexpected error")),
                    err,
                    darkred(_("skipping")),
                ),
                importance = 1,
                level = "warning",
                header = red(" @@ ")
            )
            return None
        gpg_sign_path = repo_sec.sign_file(repo, pkg_path)
        # read file content and add to 'gpg' signature
        with open(gpg_sign_path, "rb") as gpg_f:
            gpg_signature = gpg_f.read()
        os.remove(gpg_sign_path)
        return gpg_signature

    def _check_config_file_updates(self):
        self.output(
            "[%s] %s" % (
                red(_("config files")), # something short please
                blue(_("checking system")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ "),
            back = True
        )
        # scanning for config files not updated
        updates = self.ConfigurationUpdates()
        file_updates = updates.get()
        if file_updates:
            self.output(
                "[%s] %s" % (
                    red(_("config files")), # something short please
                    blue(_("there are configuration files not updated yet")),
                ),
                importance = 1,
                level = "error",
                header = darkred(" @@ ")
            )
            for val in file_updates.values():
                self.output(
                    val['destination'],
                    importance = 1,
                    level = "info",
                    header = "\t"
                )
            return True
        return False

    def _get_missing_dependencies_blacklist(self, repository_id, branch = None):
        if branch is None:
            branch = self._settings['repositories']['branch']
        wl_file = self._get_missing_dependencies_blacklist_file(repository_id,
            branch = branch)

        wl_data = set()
        enc = etpConst['conf_encoding']
        try:
            with codecs.open(wl_file, "r", encoding = enc) as f_wl:
                wl_data.update([x.strip() for x in f_wl.readlines() if
                            x.strip() and not x.strip().startswith("#")])
        except (OSError, IOError) as err:
            if err.errno != errno.ENOENT:
                raise

        return wl_data

    def _get_package_path(self, repo, dbconn, package_id):
        """
        Given EntropyRepository instance and package identifier, return local
        path of package. This method does not check for path validity though.
        """
        pkg_rel_url = dbconn.retrieveDownloadURL(package_id)
        complete_path = self.complete_local_package_path(pkg_rel_url, repo)
        return complete_path

    def _get_upload_package_path(self, repo, dbconn, package_id):
        """
        Given ServerPackagesRepository instance and package identifier,
        return local path of package.
        This method does not check for path validity though.
        """
        pkg_path = dbconn.retrieveDownloadURL(package_id)
        return os.path.join(self._get_local_upload_directory(repo), pkg_path)

    def scan_package_changes(self, repository_ids=None,
                             removal_repository_ids=None):
        """
        Scan, using Source Package Manager, for added/removed/updated
        packages.
        Please note that in order to trigger SPM level package moves,
        it is better to also call Spm().package_names_update() before
        this, or opening all the repos in read/write
        (read_only=False).

        @keyword repository_ids: list of repository identifiers to
        include in the scanning process. By default: the list of
        available repositories.
        @type repository_ids: list or iterable
        @keyword removal_repository_ids: list of repository identifiers
        to include in the list of removable packages.
        By default, only the default repository.
        @type removal_repository_ids: list or set

        @return: tuple composed of (1) list of spm package name and spm package
        id, (2) list of entropy package matches for packages to be removed (3)
        list of entropy package matches for packages to be injected.
        """
        spm = self.Spm()

        spm_packages = spm.get_installed_packages()
        installed_packages = []
        for spm_package in spm_packages:
            try:
                pkg_counter = spm.resolve_spm_package_uid(spm_package)
            except KeyError:
                # not found
                continue
            installed_packages.append((spm_package, pkg_counter,))

        installed_counters = set()
        to_be_added = set()
        to_be_removed = set()
        to_be_injected = set()
        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        exp_based_scope = my_settings['exp_based_scope']

        if repository_ids is None:
            repository_ids = self.repositories()
        if removal_repository_ids is None:
            removal_repository_ids = set([self._repository])

        # packages to be added
        for spm_atom, spm_counter in installed_packages:
            found = False
            for repository_id in repository_ids:
                installed_counters.add(spm_counter)
                repo = self.open_server_repository(
                    repository_id, read_only = True, no_upload = True)
                counter = repo.isSpmUidAvailable(spm_counter)
                if counter:
                    found = True
                    break
            if not found:
                to_be_added.add((spm_atom, spm_counter,))

        # packages to be removed from the database
        database_counters = {}
        for repository_id in repository_ids:
            repo = self.open_server_repository(
                repository_id, read_only = True, no_upload = True)
            database_counters[repository_id] = repo.listAllSpmUids()

        ordered_counters = set()
        for repository_id in database_counters:
            for data in database_counters[repository_id]:
                ordered_counters.add((data, repository_id))
        database_counters = ordered_counters

        # do some memoization to speed up the scanning
        _spm_key_slot_map = {}
        for _spm_pkg, _spm_pkg_id in to_be_added:
            key = entropy.dep.dep_getkey(_spm_pkg)
            obj = _spm_key_slot_map.setdefault(key, set())
            try:
                slot = spm.get_installed_package_metadata(
                    _spm_pkg, "SLOT")
                # workaround for ebuilds without SLOT
                if slot is None:
                    slot = "0"
                obj.add(slot)
            except KeyError:
                continue

        for (counter, package_id), repository_id in database_counters:

            if counter < 0:
                continue # skip packages without valid counter

            if counter in installed_counters:
                continue

            repo = self.open_server_repository(
                repository_id, read_only = True,
                no_upload = True)

            dorm = True
            # check if the package is in to_be_added
            if to_be_added:

                dorm = False
                atom = repo.retrieveAtom(package_id)
                atomkey = entropy.dep.dep_getkey(atom)
                atomtag = entropy.dep.dep_gettag(atom)
                atomslot = repo.retrieveSlot(package_id)
                add = True

                spm_slots = _spm_key_slot_map.get(atomkey, [])
                # atomtag != None is for handling tagged pkgs correctly
                if (atomslot in spm_slots) or (atomtag is not None):
                    # do not add to to_be_removed
                    add = False

                if not add:
                    continue
                dorm = True

            # checking if we are allowed to remove stuff on this repo
            # if repository_id is not the default one, we MUST skip this to
            # avoid touching what developer doesn't expect
            if dorm and (repository_id in removal_repository_ids):

                tag = repo.retrieveTag(package_id)
                if tag:
                    is_injected = repo.isInjected(package_id)
                    if not is_injected:
                        to_be_injected.add((package_id, repository_id))
                        continue

                trashed = self._is_spm_uid_trashed(counter)
                if trashed:
                    # search into portage then
                    try:
                        key, slot = repo.retrieveKeySlot(package_id)
                    except TypeError:
                        key, slot = None, None

                    if (key is not None) and (slot is not None):
                        slot = slot.split(",")[0]
                        try:
                            if spm.match_installed_package(key + ":" + slot):
                                trashed = True
                            else:
                                trashed = False
                        except KeyError:
                            pass

                if trashed:
                    # spm uid is trashed, and it is still installed,
                    # so there is nothing to do, really.
                    continue

                if not exp_based_scope:
                    # expiration based scope is not enabled, so
                    # the package must be accounted for removal.
                    to_be_removed.add((package_id, repository_id))
                    continue

                # check if support for this is set
                plg_id = self.sys_settings_fatscope_plugin_id
                exp_data = self._settings[plg_id]['repos'].get(
                    repository_id, set())

                # only some packages are set, check if our is
                # in the list
                if (package_id not in exp_data) and (-1 not in exp_data):
                    to_be_removed.add((package_id, repository_id))
                    continue

                package_id_expired = self._is_match_expired((package_id,
                    repository_id,))

                if package_id_expired:
                    # expired !!!
                    # add this and its depends (reverse deps)

                    rm_match = (package_id, repository_id)
                    #to_be_removed.add(rm_match)
                    revdep_matches = self.get_reverse_queue([rm_match],
                        system_packages = False)
                    to_be_removed.update(revdep_matches)

        return to_be_added, to_be_removed, to_be_injected

    def _is_match_expired(self, match):

        package_id, repoid = match
        dbconn = self.open_server_repository(repoid, just_reading = True)
        # 3600 * 24 = 86400
        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        pkg_exp_secs = my_settings['packages_expiration_days'] * 86400
        cur_unix_time = time.time()
        # if packages removal is triggered by expiration
        # we will have to check if our package is really
        # expired and remove its reverse deps too
        mydate = dbconn.retrieveCreationDate(package_id)
        # cross fingers hoping that time is set correctly
        mydelta = cur_unix_time - float(mydate)
        if mydelta > pkg_exp_secs:
            return True
        return False

    def _is_spm_uid_trashed(self, counter):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        server_repos = list(srv_set['repositories'].keys())
        for repo in server_repos:
            dbconn = self.open_server_repository(repo, read_only = True,
                no_upload = True)
            if dbconn.isSpmUidTrashed(counter):
                return True
        return False

    def _transform_package_into_injected(self, package_id, repository_id):
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        counter = dbconn.getFakeSpmUid()
        dbconn.setSpmUid(package_id, counter)
        dbconn.setInjected(package_id)
        dbconn.commit()

    def _pump_extracted_package_metadata(self, pkg_meta, repo, extra_metadata):
        """
        Add to pkg_meta dict, server-side package metadata information before
        injecting it into repository database.
        """
        # add extra metadata
        pkg_meta.update(extra_metadata)
        # do not set GPG signature here for performance, just provide an
        # empty default. Real GPG signature will be written inside
        # _inject_database_into_packages()
        pkg_meta['signatures']['gpg'] = None

        # rewrite dependency strings using dep_rewrite metadata
        self.__handle_dep_rewrite(pkg_meta, repo)

    def __handle_dep_rewrite(self, pkg_meta, repo):

        dep_rewrite = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['dep_rewrite']

        # NOTE: to be able to match the package, we need to add it
        # to a temp repo
        tmp_repo = self._open_temp_repository("dep_rewrite_temp",
            temp_file = ":memory:")
        new_package_id = tmp_repo.handlePackage(pkg_meta)
        pkg_atom = tmp_repo.retrieveAtom(new_package_id)

        rewrites_enabled = []
        wildcard_rewrite = False
        for dep_string_rewrite, dep_pattern in dep_rewrite:
            # magic catch-all support
            if dep_string_rewrite == "*":
                pkg_id, rc = None, 0
                wildcard_rewrite = True
            else:
                wildcard_rewrite = False
                pkg_id, rc = tmp_repo.atomMatch(dep_string_rewrite)
            if rc == 0:
                rewrites_enabled.append((dep_string_rewrite, dep_pattern))
        tmp_repo.close()

        if not rewrites_enabled:
            return

        if not wildcard_rewrite:
            self.output(
                "[%s|%s] %s:" % (
                        blue(repo),
                        brown(pkg_atom),
                        teal(
                            _("found available dep_rewrites for this package")),
                    ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            for dep_string_rewrite, dep_pattern in rewrites_enabled:
                compiled_pattern, replaces = \
                    dep_rewrite[(dep_string_rewrite, dep_pattern)]
                if compiled_pattern is None:
                    # this means that user is asking to add dep_pattern
                    # as a dependency to package
                    replaces_str = _("added")
                elif not replaces:
                    # this means that user is asking to remove dep_pattern
                    replaces_str = _("removed")
                else:
                    replaces_str = "=> " + ', '.join(replaces)
                self.output(
                    "%s %s" % (
                        purple(dep_pattern),
                        replaces_str,
                    ),
                    importance = 1,
                    level = "info",
                    header = teal("   # ")
                )

        def _extract_dep_add_from_dep_pattern(_dep_pattern):
            """
            when action is to add a dependency, extracts the dependency type
            from dependency string, if found.
            """
            _dep_type = etpConst['dependency_type_ids']['mdepend_id']
            bracket_start_idx = _dep_pattern.rfind("<")
            _is_conflict = False

            if _dep_pattern.endswith(">") and (bracket_start_idx != -1):
                _c_dep_type = _dep_pattern[bracket_start_idx+1:-1]
                try:
                    _c_dep_type = int(_c_dep_type)
                    if _c_dep_type not in (1, 2, 3, 4):
                        raise ValueError()
                except ValueError:
                    # cannot correctly evaluate, giving up silently
                    return _dep_pattern, _dep_type

                _dep_pattern = _dep_pattern[:bracket_start_idx]
                # given packages.server.dep_rewrite.example specifications
                # this is the mapping:
                if _c_dep_type == 1:
                    _dep_type = etpConst['dependency_type_ids']['bdepend_id']
                elif _c_dep_type == 2:
                    _dep_type = etpConst['dependency_type_ids']['rdepend_id']
                elif _c_dep_type == 3:
                    _dep_type = etpConst['dependency_type_ids']['pdepend_id']
                elif _c_dep_type == 4:
                    # manual dependency
                    _dep_type = etpConst['dependency_type_ids']['mdepend_id']

            if _dep_pattern.startswith("!"):
                _is_conflict = True
                _dep_pattern = _dep_pattern[1:]

            return _dep_pattern, _dep_type, _is_conflict

        # pkg_meta['conflicts'] is a frozenset
        conflicts = set(pkg_meta['conflicts'])
        new_dependencies = []
        remove_dependencies = set()

        for dep_string, dep_value in pkg_meta['pkg_dependencies']:

            dep_string_matched = False
            matched_pattern = False

            for key in rewrites_enabled:

                dep_string_rewrite, dep_pattern = key
                compiled_pattern, replaces = dep_rewrite[key]
                if compiled_pattern is None:
                    # user is asking to add dep_pattern to dependency list
                    dep_pattern_string, dep_pattern_type, conflict = \
                        _extract_dep_add_from_dep_pattern(dep_pattern)
                    if conflict:
                        conflicts.add(dep_pattern_string)
                    else:
                        new_dependencies.append(
                            (dep_pattern_string, dep_pattern_type))
                    continue

                if not compiled_pattern.match(dep_string):
                    # dep_string not matched, skipping
                    continue
                matched_pattern = True

                if not replaces:
                    # then it's a removal
                    dep_string_matched = True

                for replace in replaces:
                    new_dep_string, number_of_subs_made = \
                        compiled_pattern.subn(replace, dep_string)
                    if number_of_subs_made:
                        dep_string_matched = True
                        if new_dep_string and (new_dep_string != "-"):
                            new_dependencies.append((new_dep_string, dep_value))
                            self.output(
                                "%s: %s => %s" % (
                                    teal(_("replaced")),
                                    brown(dep_string),
                                    purple(new_dep_string),
                                ),
                                importance = 1,
                                level = "info",
                                header = purple("   ! ")
                            )
                        else:
                            self.output(
                                "%s: %s => X" % (
                                    teal(_("removed")),
                                    brown(dep_string),
                                ),
                                importance = 1,
                                level = "info",
                                header = purple("   ! ")
                            )

                    else:
                        self.output(
                            "%s: %s + %s" % (
                                darkred(_("No dependency rewrite made for")),
                                brown(dep_string),
                                purple(replace),
                            ),
                            importance = 1,
                            level = "warning",
                            header = darkred("   !!! ")
                        )

            if dep_string_matched:
                remove_dependencies.add(dep_string)
            elif (not dep_string_matched) and matched_pattern:
                self.output(
                    "%s: %s :: %s" % (
                        darkred(_("No dependency rewrite made for")),
                        brown(pkg_atom),
                        purple(dep_string),
                    ),
                    importance = 1,
                    level = "warning",
                    header = darkred("   !x!x!x! ")
                )

        pkg_dependencies = []
        for dep_string, dep_value in pkg_meta['pkg_dependencies']:
            if dep_string in remove_dependencies:
                continue
            pkg_dependencies.append((dep_string, dep_value))

        pkg_dependencies.extend(new_dependencies)
        pkg_meta['pkg_dependencies'] = tuple(pkg_dependencies)

        # save conflicts metadata back in place
        pkg_meta['conflicts'] = frozenset(conflicts)

    def _get_entropy_sets(self, repository_id, branch = None):
        if branch is None:
            branch = self._settings['repositories']['branch']
        sets_dir = self._get_local_database_sets_dir(repository_id,
            branch = branch)

        items = []
        try:
            items += os.listdir(sets_dir)
        except (OSError, IOError) as err:
            if err.errno != errno.ENOENT:
                raise

        mydata = {}
        for item in items:

            try:
                item_clean = str(item)
            except (UnicodeEncodeError, UnicodeDecodeError,):
                continue
            item_path = os.path.join(sets_dir, item)

            try:
                item_elements = self._settings._extract_packages_from_set_file(
                    item_path)
            except (OSError, IOError) as err:
                if err.errno == errno.ENOENT:
                    continue
                raise

            if item_elements:
                mydata[item_clean] = item_elements.copy()

        return mydata

    def _get_configured_package_sets(self, repository_id):

        branch = self._settings['repositories']['branch']
        # portage sets
        sets_data = self.Spm().get_package_sets(False)
        sets_data.update(self._get_entropy_sets(repository_id, branch = branch))

        invalid_sets = set()
        # validate
        for setname in sets_data:
            good = True
            for atom in sets_data[setname]:
                if atom.startswith(etpConst['packagesetprefix']):
                    # ignore nested package sets
                    continue
                dbconn = self.open_server_repository(repository_id,
                    just_reading = True)
                match = dbconn.atomMatch(atom)
                if match[0] == -1:
                    good = False
                    break
            if not good:
                invalid_sets.add(setname)
        for invalid_set in invalid_sets:
            del sets_data[invalid_set]

        return sets_data

    def _update_package_sets(self, repoitory_id, entropy_repository):

        self.output(
            "[%s|%s] %s..." % (
                darkgreen(repoitory_id),
                purple(_("sets")),
                blue(_("updating package sets")),
            ),
            importance = 0,
            level = "info",
            header = blue(" @@ "),
            back = True
        )

        package_sets = self._get_configured_package_sets(repoitory_id)

        # tell what package sets got added, and what got removed
        current_sets = set(entropy_repository.retrievePackageSets())
        configured_sets = set(package_sets)
        new_sets = sorted(configured_sets - current_sets)
        removed_sets = sorted(current_sets - configured_sets)
        for new_set in new_sets:
            self.output(
                "[%s|%s] %s: %s" % (
                    darkgreen(repoitory_id),
                    purple(_("sets")),
                    blue(_("adding package set")),
                    brown(new_set),
                ),
                importance = 0,
                level = "info",
                header = darkgreen(" @@ ")
            )
        for removed_set in removed_sets:
            self.output(
                "[%s|%s] %s: %s" % (
                    teal(repoitory_id),
                    brown(_("sets")),
                    purple(_("removing package set")),
                    bold(removed_set),
                ),
                importance = 1,
                level = "warning",
                header = darkred(" @@ ")
            )

        entropy_repository.clearPackageSets()
        if package_sets:
            entropy_repository.insertPackageSets(package_sets)
