Current File : //usr/lib/python3.6/site-packages/dnf/base.py
# Copyright 2005 Duke University
# Copyright (C) 2012-2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
Supplies the Base class.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import dnf
import libdnf.transaction
from copy import deepcopy
from dnf.comps import CompsQuery
from dnf.i18n import _, P_, ucd
from dnf.util import _parse_specs
from dnf.db.history import SwdbInterface
from dnf.yum import misc
try:
from collections.abc import Sequence
except ImportError:
from collections import Sequence
import datetime
import dnf.callback
import dnf.comps
import dnf.conf
import dnf.conf.read
import dnf.crypto
import dnf.dnssec
import dnf.drpm
import dnf.exceptions
import dnf.goal
import dnf.history
import dnf.lock
import dnf.logging
# WITH_MODULES is used by ansible (lib/ansible/modules/packaging/os/dnf.py)
try:
import dnf.module.module_base
WITH_MODULES = True
except ImportError:
WITH_MODULES = False
import dnf.persistor
import dnf.plugin
import dnf.query
import dnf.repo
import dnf.repodict
import dnf.rpm.connection
import dnf.rpm.miscutils
import dnf.rpm.transaction
import dnf.sack
import dnf.selector
import dnf.subject
import dnf.transaction
import dnf.util
import dnf.yum.rpmtrans
import functools
import gc
import hawkey
import itertools
import logging
import math
import os
import operator
import re
import rpm
import time
import shutil
logger = logging.getLogger("dnf")
class Base(object):
def __init__(self, conf=None):
# :api
self._closed = False
self._conf = conf or self._setup_default_conf()
self._goal = None
self._repo_persistor = None
self._sack = None
self._transaction = None
self._priv_ts = None
self._comps = None
self._comps_trans = dnf.comps.TransactionBunch()
self._history = None
self._tempfiles = set()
self._trans_tempfiles = set()
self._ds_callback = dnf.callback.Depsolve()
self._logging = dnf.logging.Logging()
self._repos = dnf.repodict.RepoDict()
self._rpm_probfilter = set([rpm.RPMPROB_FILTER_OLDPACKAGE])
self._plugins = dnf.plugin.Plugins()
self._trans_success = False
self._trans_install_set = False
self._tempfile_persistor = None
# self._update_security_filters is used by ansible
self._update_security_filters = []
self._update_security_options = {}
self._allow_erasing = False
self._repo_set_imported_gpg_keys = set()
self.output = None
def __enter__(self):
return self
def __exit__(self, *exc_args):
self.close()
def __del__(self):
self.close()
def _add_tempfiles(self, files):
if self._transaction:
self._trans_tempfiles.update(files)
elif self.conf.destdir:
pass
else:
self._tempfiles.update(files)
def _add_repo_to_sack(self, repo):
repo.load()
mdload_flags = dict(load_filelists=True,
load_presto=repo.deltarpm,
load_updateinfo=True)
if repo.load_metadata_other:
mdload_flags["load_other"] = True
try:
self._sack.load_repo(repo._repo, build_cache=True, **mdload_flags)
except hawkey.Exception as e:
logger.debug(_("loading repo '{}' failure: {}").format(repo.id, e))
raise dnf.exceptions.RepoError(
_("Loading repository '{}' has failed").format(repo.id))
@staticmethod
def _setup_default_conf():
conf = dnf.conf.Conf()
subst = conf.substitutions
if 'releasever' not in subst:
subst['releasever'] = \
dnf.rpm.detect_releasever(conf.installroot)
return conf
def _setup_modular_excludes(self):
hot_fix_repos = [i.id for i in self.repos.iter_enabled() if i.module_hotfixes]
try:
solver_errors = self.sack.filter_modules(
self._moduleContainer, hot_fix_repos, self.conf.installroot,
self.conf.module_platform_id, update_only=False, debugsolver=self.conf.debug_solver,
module_obsoletes=self.conf.module_obsoletes)
except hawkey.Exception as e:
raise dnf.exceptions.Error(ucd(e))
if solver_errors:
logger.warning(
dnf.module.module_base.format_modular_solver_errors(solver_errors[0]))
def _setup_excludes_includes(self, only_main=False):
disabled = set(self.conf.disable_excludes)
if 'all' in disabled and WITH_MODULES:
self._setup_modular_excludes()
return
repo_includes = []
repo_excludes = []
# first evaluate repo specific includes/excludes
if not only_main:
for r in self.repos.iter_enabled():
if r.id in disabled:
continue
if len(r.includepkgs) > 0:
incl_query = self.sack.query().filterm(empty=True)
for incl in set(r.includepkgs):
subj = dnf.subject.Subject(incl)
incl_query = incl_query.union(subj.get_best_query(
self.sack, with_nevra=True, with_provides=False, with_filenames=False))
incl_query.filterm(reponame=r.id)
repo_includes.append((incl_query.apply(), r.id))
excl_query = self.sack.query().filterm(empty=True)
for excl in set(r.excludepkgs):
subj = dnf.subject.Subject(excl)
excl_query = excl_query.union(subj.get_best_query(
self.sack, with_nevra=True, with_provides=False, with_filenames=False))
excl_query.filterm(reponame=r.id)
if excl_query:
repo_excludes.append((excl_query, r.id))
# then main (global) includes/excludes because they can mask
# repo specific settings
if 'main' not in disabled:
include_query = self.sack.query().filterm(empty=True)
if len(self.conf.includepkgs) > 0:
for incl in set(self.conf.includepkgs):
subj = dnf.subject.Subject(incl)
include_query = include_query.union(subj.get_best_query(
self.sack, with_nevra=True, with_provides=False, with_filenames=False))
exclude_query = self.sack.query().filterm(empty=True)
for excl in set(self.conf.excludepkgs):
subj = dnf.subject.Subject(excl)
exclude_query = exclude_query.union(subj.get_best_query(
self.sack, with_nevra=True, with_provides=False, with_filenames=False))
if len(self.conf.includepkgs) > 0:
self.sack.add_includes(include_query)
self.sack.set_use_includes(True)
if exclude_query:
self.sack.add_excludes(exclude_query)
if repo_includes:
for query, repoid in repo_includes:
self.sack.add_includes(query)
self.sack.set_use_includes(True, repoid)
if repo_excludes:
for query, repoid in repo_excludes:
self.sack.add_excludes(query)
if not only_main and WITH_MODULES:
self._setup_modular_excludes()
def _store_persistent_data(self):
if self._repo_persistor and not self.conf.cacheonly:
expired = [r.id for r in self.repos.iter_enabled()
if (r.metadata and r._repo.isExpired())]
self._repo_persistor.expired_to_add.update(expired)
self._repo_persistor.save()
if self._tempfile_persistor:
self._tempfile_persistor.save()
@property
def comps(self):
# :api
if self._comps is None:
self.read_comps(arch_filter=True)
return self._comps
@property
def conf(self):
# :api
return self._conf
@property
def repos(self):
# :api
return self._repos
@repos.deleter
def repos(self):
# :api
self._repos = None
@property
@dnf.util.lazyattr("_priv_rpmconn")
def _rpmconn(self):
return dnf.rpm.connection.RpmConnection(self.conf.installroot)
@property
def sack(self):
# :api
return self._sack
@property
def _moduleContainer(self):
if self.sack is None:
raise dnf.exceptions.Error("Sack was not initialized")
if self.sack._moduleContainer is None:
self.sack._moduleContainer = libdnf.module.ModulePackageContainer(
False, self.conf.installroot, self.conf.substitutions["arch"], self.conf.persistdir)
return self.sack._moduleContainer
@property
def transaction(self):
# :api
return self._transaction
@transaction.setter
def transaction(self, value):
# :api
if self._transaction:
raise ValueError('transaction already set')
self._transaction = value
def _activate_persistor(self):
self._repo_persistor = dnf.persistor.RepoPersistor(self.conf.cachedir)
def init_plugins(self, disabled_glob=(), enable_plugins=(), cli=None):
# :api
"""Load plugins and run their __init__()."""
if self.conf.plugins:
self._plugins._load(self.conf, disabled_glob, enable_plugins)
self._plugins._run_init(self, cli)
def pre_configure_plugins(self):
# :api
"""Run plugins pre_configure() method."""
self._plugins._run_pre_config()
def configure_plugins(self):
# :api
"""Run plugins configure() method."""
self._plugins._run_config()
def unload_plugins(self):
# :api
"""Run plugins unload() method."""
self._plugins._unload()
def update_cache(self, timer=False):
# :api
period = self.conf.metadata_timer_sync
if self._repo_persistor is None:
self._activate_persistor()
persistor = self._repo_persistor
if timer:
if dnf.util.on_metered_connection():
msg = _('Metadata timer caching disabled '
'when running on metered connection.')
logger.info(msg)
return False
if dnf.util.on_ac_power() is False:
msg = _('Metadata timer caching disabled '
'when running on a battery.')
logger.info(msg)
return False
if period <= 0:
msg = _('Metadata timer caching disabled.')
logger.info(msg)
return False
since_last_makecache = persistor.since_last_makecache()
if since_last_makecache is not None and since_last_makecache < period:
logger.info(_('Metadata cache refreshed recently.'))
return False
for repo in self.repos.values():
repo._repo.setMaxMirrorTries(1)
if not self.repos._any_enabled():
logger.info(_('There are no enabled repositories in "{}".').format(
'", "'.join(self.conf.reposdir)))
return False
for r in self.repos.iter_enabled():
(is_cache, expires_in) = r._metadata_expire_in()
if expires_in is None:
logger.info(_('%s: will never be expired and will not be refreshed.'), r.id)
elif not is_cache or expires_in <= 0:
logger.debug(_('%s: has expired and will be refreshed.'), r.id)
r._repo.expire()
elif timer and expires_in < period:
# expires within the checking period:
msg = _("%s: metadata will expire after %d seconds and will be refreshed now")
logger.debug(msg, r.id, expires_in)
r._repo.expire()
else:
logger.debug(_('%s: will expire after %d seconds.'), r.id,
expires_in)
if timer:
persistor.reset_last_makecache = True
self.fill_sack(load_system_repo=False, load_available_repos=True) # performs the md sync
logger.info(_('Metadata cache created.'))
return True
def fill_sack(self, load_system_repo=True, load_available_repos=True):
# :api
"""Prepare the Sack and the Goal objects. """
timer = dnf.logging.Timer('sack setup')
self.reset(sack=True, goal=True)
self._sack = dnf.sack._build_sack(self)
lock = dnf.lock.build_metadata_lock(self.conf.cachedir, self.conf.exit_on_lock)
with lock:
if load_system_repo is not False:
try:
# FIXME: If build_cache=True, @System.solv is incorrectly updated in install-
# remove loops
self._sack.load_system_repo(build_cache=False)
except IOError:
if load_system_repo != 'auto':
raise
if load_available_repos:
error_repos = []
mts = 0
age = time.time()
# Iterate over installed GPG keys and check their validity using DNSSEC
if self.conf.gpgkey_dns_verification:
dnf.dnssec.RpmImportedKeys.check_imported_keys_validity()
for r in self.repos.iter_enabled():
try:
self._add_repo_to_sack(r)
if r._repo.getTimestamp() > mts:
mts = r._repo.getTimestamp()
if r._repo.getAge() < age:
age = r._repo.getAge()
logger.debug(_("%s: using metadata from %s."), r.id,
dnf.util.normalize_time(
r._repo.getMaxTimestamp()))
except dnf.exceptions.RepoError as e:
r._repo.expire()
if r.skip_if_unavailable is False:
raise
logger.warning("Error: %s", e)
error_repos.append(r.id)
r.disable()
if error_repos:
logger.warning(
_("Ignoring repositories: %s"), ', '.join(error_repos))
if self.repos._any_enabled():
if age != 0 and mts != 0:
logger.info(_("Last metadata expiration check: %s ago on %s."),
datetime.timedelta(seconds=int(age)),
dnf.util.normalize_time(mts))
else:
self.repos.all().disable()
conf = self.conf
self._sack._configure(conf.installonlypkgs, conf.installonly_limit, conf.allow_vendor_change)
self._setup_excludes_includes()
timer()
self._goal = dnf.goal.Goal(self._sack)
self._goal.protect_running_kernel = conf.protect_running_kernel
self._plugins.run_sack()
return self._sack
def fill_sack_from_repos_in_cache(self, load_system_repo=True):
# :api
"""
Prepare Sack and Goal objects and also load all enabled repositories from cache only,
it doesn't download anything and it doesn't check if metadata are expired.
If there is not enough metadata present (repond.xml or both primary.xml and solv file
are missing) given repo is either skipped or it throws a RepoError exception depending
on skip_if_unavailable configuration.
"""
timer = dnf.logging.Timer('sack setup')
self.reset(sack=True, goal=True)
self._sack = dnf.sack._build_sack(self)
lock = dnf.lock.build_metadata_lock(self.conf.cachedir, self.conf.exit_on_lock)
with lock:
if load_system_repo is not False:
try:
# FIXME: If build_cache=True, @System.solv is incorrectly updated in install-
# remove loops
self._sack.load_system_repo(build_cache=False)
except IOError:
if load_system_repo != 'auto':
raise
error_repos = []
# Iterate over installed GPG keys and check their validity using DNSSEC
if self.conf.gpgkey_dns_verification:
dnf.dnssec.RpmImportedKeys.check_imported_keys_validity()
for repo in self.repos.iter_enabled():
try:
repo._repo.loadCache(throwExcept=True, ignoreMissing=True)
mdload_flags = dict(load_filelists=True,
load_presto=repo.deltarpm,
load_updateinfo=True)
if repo.load_metadata_other:
mdload_flags["load_other"] = True
self._sack.load_repo(repo._repo, **mdload_flags)
logger.debug(_("%s: using metadata from %s."), repo.id,
dnf.util.normalize_time(
repo._repo.getMaxTimestamp()))
except (RuntimeError, hawkey.Exception) as e:
if repo.skip_if_unavailable is False:
raise dnf.exceptions.RepoError(
_("loading repo '{}' failure: {}").format(repo.id, e))
else:
logger.debug(_("loading repo '{}' failure: {}").format(repo.id, e))
error_repos.append(repo.id)
repo.disable()
if error_repos:
logger.warning(
_("Ignoring repositories: %s"), ', '.join(error_repos))
conf = self.conf
self._sack._configure(conf.installonlypkgs, conf.installonly_limit, conf.allow_vendor_change)
self._setup_excludes_includes()
timer()
self._goal = dnf.goal.Goal(self._sack)
self._goal.protect_running_kernel = conf.protect_running_kernel
self._plugins.run_sack()
return self._sack
def _finalize_base(self):
self._tempfile_persistor = dnf.persistor.TempfilePersistor(
self.conf.cachedir)
if not self.conf.keepcache:
self._clean_packages(self._tempfiles)
if self._trans_success:
self._trans_tempfiles.update(
self._tempfile_persistor.get_saved_tempfiles())
self._tempfile_persistor.empty()
if self._trans_install_set:
self._clean_packages(self._trans_tempfiles)
else:
self._tempfile_persistor.tempfiles_to_add.update(
self._trans_tempfiles)
if self._tempfile_persistor.tempfiles_to_add:
logger.info(_("The downloaded packages were saved in cache "
"until the next successful transaction."))
logger.info(_("You can remove cached packages by executing "
"'%s'."), "{prog} clean packages".format(prog=dnf.util.MAIN_PROG))
# Do not trigger the lazy creation:
if self._history is not None:
self.history.close()
self._store_persistent_data()
self._closeRpmDB()
self._trans_success = False
def close(self):
# :api
"""Close all potential handles and clean cache.
Typically the handles are to data sources and sinks.
"""
if self._closed:
return
logger.log(dnf.logging.DDEBUG, 'Cleaning up.')
self._closed = True
self._finalize_base()
self.reset(sack=True, repos=True, goal=True)
self._plugins = None
def read_all_repos(self, opts=None):
# :api
"""Read repositories from the main conf file and from .repo files."""
reader = dnf.conf.read.RepoReader(self.conf, opts)
for repo in reader:
try:
self.repos.add(repo)
except dnf.exceptions.ConfigError as e:
logger.warning(e)
def reset(self, sack=False, repos=False, goal=False):
# :api
"""Make the Base object forget about various things."""
if sack:
self._sack = None
if repos:
self._repos = dnf.repodict.RepoDict()
if goal:
self._goal = None
if self._sack is not None:
self._goal = dnf.goal.Goal(self._sack)
self._goal.protect_running_kernel = self.conf.protect_running_kernel
if self._sack and self._moduleContainer:
# sack must be set to enable operations on moduleContainer
self._moduleContainer.rollback()
if self._history is not None:
self.history.close()
self._comps_trans = dnf.comps.TransactionBunch()
self._transaction = None
self._update_security_filters = []
if sack and goal:
# We've just done this, above:
#
# _sack _goal
# | |
# -- [CUT] -- -- [CUT] --
# | |
# v | v
# +----------------+ [C] +-------------+
# | DnfSack object | <-[U]- | Goal object |
# +----------------+ [T] +-------------+
# |^ |^ |^ |
# || || ||
# || || || |
# +--||----||----||---+ [C]
# | v| v| v| | <--[U]-- _transaction
# | Pkg1 Pkg2 PkgN | [T]
# | | |
# | Transaction oject |
# +-------------------+
#
# At this point, the DnfSack object would be released only
# eventually, by Python's generational garbage collector, due to the
# cyclic references DnfSack<->Pkg1 ... DnfSack<->PkgN.
#
# The delayed release is a problem: the DnfSack object may
# (indirectly) own "page file" file descriptors in libsolv, via
# libdnf. For example,
#
# sack->priv->pool->repos[1]->repodata[1]->store.pagefd = 7
# sack->priv->pool->repos[1]->repodata[2]->store.pagefd = 8
#
# These file descriptors are closed when the DnfSack object is
# eventually released, that is, when dnf_sack_finalize() (in libdnf)
# calls pool_free() (in libsolv).
#
# We need that to happen right now, as callers may want to unmount
# the filesystems which those file descriptors refer to immediately
# after reset() returns. Therefore, force a garbage collection here.
gc.collect()
def _closeRpmDB(self):
"""Closes down the instances of rpmdb that could be open."""
del self._ts
_TS_FLAGS_TO_RPM = {'noscripts': rpm.RPMTRANS_FLAG_NOSCRIPTS,
'notriggers': rpm.RPMTRANS_FLAG_NOTRIGGERS,
'nodocs': rpm.RPMTRANS_FLAG_NODOCS,
'test': rpm.RPMTRANS_FLAG_TEST,
'justdb': rpm.RPMTRANS_FLAG_JUSTDB,
'nocontexts': rpm.RPMTRANS_FLAG_NOCONTEXTS,
'nocrypto': rpm.RPMTRANS_FLAG_NOFILEDIGEST}
if hasattr(rpm, 'RPMTRANS_FLAG_NOCAPS'):
# Introduced in rpm-4.14
_TS_FLAGS_TO_RPM['nocaps'] = rpm.RPMTRANS_FLAG_NOCAPS
_TS_VSFLAGS_TO_RPM = {'nocrypto': rpm._RPMVSF_NOSIGNATURES |
rpm._RPMVSF_NODIGESTS}
@property
def goal(self):
return self._goal
@property
def _ts(self):
"""Set up the RPM transaction set that will be used
for all the work."""
if self._priv_ts is not None:
return self._priv_ts
self._priv_ts = dnf.rpm.transaction.TransactionWrapper(
self.conf.installroot)
self._priv_ts.setFlags(0) # reset everything.
for flag in self.conf.tsflags:
rpm_flag = self._TS_FLAGS_TO_RPM.get(flag)
if rpm_flag is None:
logger.critical(_('Invalid tsflag in config file: %s'), flag)
continue
self._priv_ts.addTsFlag(rpm_flag)
vs_flag = self._TS_VSFLAGS_TO_RPM.get(flag)
if vs_flag is not None:
self._priv_ts.pushVSFlags(vs_flag)
if not self.conf.diskspacecheck:
self._rpm_probfilter.add(rpm.RPMPROB_FILTER_DISKSPACE)
if self.conf.ignorearch:
self._rpm_probfilter.add(rpm.RPMPROB_FILTER_IGNOREARCH)
probfilter = functools.reduce(operator.or_, self._rpm_probfilter, 0)
self._priv_ts.setProbFilter(probfilter)
return self._priv_ts
@_ts.deleter
def _ts(self):
"""Releases the RPM transaction set. """
if self._priv_ts is None:
return
self._priv_ts.close()
del self._priv_ts
self._priv_ts = None
def read_comps(self, arch_filter=False):
# :api
"""Create the groups object to access the comps metadata."""
timer = dnf.logging.Timer('loading comps')
self._comps = dnf.comps.Comps()
logger.log(dnf.logging.DDEBUG, 'Getting group metadata')
for repo in self.repos.iter_enabled():
if not repo.enablegroups:
continue
if not repo.metadata:
continue
comps_fn = repo._repo.getCompsFn()
if not comps_fn:
continue
logger.log(dnf.logging.DDEBUG,
'Adding group file from repository: %s', repo.id)
if repo._repo.getSyncStrategy() == dnf.repo.SYNC_ONLY_CACHE:
decompressed = misc.calculate_repo_gen_dest(comps_fn,
'groups.xml')
if not os.path.exists(decompressed):
# root privileges are needed for comps decompression
continue
else:
decompressed = misc.repo_gen_decompress(comps_fn, 'groups.xml')
try:
self._comps._add_from_xml_filename(decompressed)
except dnf.exceptions.CompsError as e:
msg = _('Failed to add groups file for repository: %s - %s')
logger.critical(msg, repo.id, e)
if arch_filter:
self._comps._i.arch_filter(
[self._conf.substitutions['basearch']])
timer()
return self._comps
def _getHistory(self):
"""auto create the history object that to access/append the transaction
history information. """
if self._history is None:
releasever = self.conf.releasever
self._history = SwdbInterface(self.conf.persistdir, releasever=releasever)
return self._history
history = property(fget=lambda self: self._getHistory(),
fset=lambda self, value: setattr(
self, "_history", value),
fdel=lambda self: setattr(self, "_history", None),
doc="DNF SWDB Interface Object")
def _goal2transaction(self, goal):
ts = self.history.rpm
all_obsoleted = set(goal.list_obsoleted())
installonly_query = self._get_installonly_query()
installonly_query.apply()
installonly_query_installed = installonly_query.installed().apply()
for pkg in goal.list_downgrades():
obs = goal.obsoleted_by_package(pkg)
downgraded = obs[0]
self._ds_callback.pkg_added(downgraded, 'dd')
self._ds_callback.pkg_added(pkg, 'd')
ts.add_downgrade(pkg, downgraded, obs[1:])
for pkg in goal.list_reinstalls():
self._ds_callback.pkg_added(pkg, 'r')
obs = goal.obsoleted_by_package(pkg)
nevra_pkg = str(pkg)
# reinstall could obsolete multiple packages with the same NEVRA or different NEVRA
# Set the package with the same NEVRA as reinstalled
obsoletes = []
for obs_pkg in obs:
if str(obs_pkg) == nevra_pkg:
obsoletes.insert(0, obs_pkg)
else:
obsoletes.append(obs_pkg)
reinstalled = obsoletes[0]
ts.add_reinstall(pkg, reinstalled, obsoletes[1:])
for pkg in goal.list_installs():
self._ds_callback.pkg_added(pkg, 'i')
obs = goal.obsoleted_by_package(pkg)
# Skip obsoleted packages that are not part of all_obsoleted,
# they are handled as upgrades/downgrades.
# Also keep RPMs with the same name - they're not always in all_obsoleted.
obs = [i for i in obs if i in all_obsoleted or i.name == pkg.name]
reason = goal.get_reason(pkg)
# Inherit reason if package is installonly an package with same name is installed
# Use the same logic like upgrade
# Upgrade of installonly packages result in install or install and remove step
if pkg in installonly_query and installonly_query_installed.filter(name=pkg.name):
reason = ts.get_reason(pkg)
# inherit the best reason from obsoleted packages
for obsolete in obs:
reason_obsolete = ts.get_reason(obsolete)
if libdnf.transaction.TransactionItemReasonCompare(reason, reason_obsolete) == -1:
reason = reason_obsolete
ts.add_install(pkg, obs, reason)
cb = lambda pkg: self._ds_callback.pkg_added(pkg, 'od')
dnf.util.mapall(cb, obs)
for pkg in goal.list_upgrades():
obs = goal.obsoleted_by_package(pkg)
upgraded = None
for i in obs:
# try to find a package with matching name as the upgrade
if i.name == pkg.name:
upgraded = i
break
if upgraded is None:
# no matching name -> pick the first one
upgraded = obs.pop(0)
else:
obs.remove(upgraded)
# Skip obsoleted packages that are not part of all_obsoleted,
# they are handled as upgrades/downgrades.
# Also keep RPMs with the same name - they're not always in all_obsoleted.
obs = [i for i in obs if i in all_obsoleted or i.name == pkg.name]
cb = lambda pkg: self._ds_callback.pkg_added(pkg, 'od')
dnf.util.mapall(cb, obs)
if pkg in installonly_query:
ts.add_install(pkg, obs)
else:
ts.add_upgrade(pkg, upgraded, obs)
self._ds_callback.pkg_added(upgraded, 'ud')
self._ds_callback.pkg_added(pkg, 'u')
erasures = goal.list_erasures()
if erasures:
remaining_installed_query = self.sack.query(flags=hawkey.IGNORE_EXCLUDES).installed()
remaining_installed_query.filterm(pkg__neq=erasures)
for pkg in erasures:
if remaining_installed_query.filter(name=pkg.name):
remaining = remaining_installed_query[0]
ts.get_reason(remaining)
self.history.set_reason(remaining, ts.get_reason(remaining))
self._ds_callback.pkg_added(pkg, 'e')
reason = goal.get_reason(pkg)
ts.add_erase(pkg, reason)
return ts
def _query_matches_installed(self, q):
""" See what packages in the query match packages (also in older
versions, but always same architecture) that are already installed.
Unlike in case of _sltr_matches_installed(), it is practical here
to know even the packages in the original query that can still be
installed.
"""
inst = q.installed()
inst_per_arch = inst._na_dict()
avail_per_arch = q.available()._na_dict()
avail_l = []
inst_l = []
for na in avail_per_arch:
if na in inst_per_arch:
inst_l.append(inst_per_arch[na][0])
else:
avail_l.append(avail_per_arch[na])
return inst_l, avail_l
def _sltr_matches_installed(self, sltr):
""" See if sltr matches a patches that is (in older version or different
architecture perhaps) already installed.
"""
inst = self.sack.query().installed().filterm(pkg=sltr.matches())
return list(inst)
def iter_userinstalled(self):
"""Get iterator over the packages installed by the user."""
return (pkg for pkg in self.sack.query().installed()
if self.history.user_installed(pkg))
def _run_hawkey_goal(self, goal, allow_erasing):
ret = goal.run(
allow_uninstall=allow_erasing, force_best=self.conf.best,
ignore_weak_deps=(not self.conf.install_weak_deps))
if self.conf.debug_solver:
goal.write_debugdata('./debugdata/rpms')
return ret
def resolve(self, allow_erasing=False):
# :api
"""Build the transaction set."""
exc = None
self._finalize_comps_trans()
timer = dnf.logging.Timer('depsolve')
self._ds_callback.start()
goal = self._goal
if goal.req_has_erase():
goal.push_userinstalled(self.sack.query().installed(),
self.history)
elif not self.conf.upgrade_group_objects_upgrade:
# exclude packages installed from groups
# these packages will be marked to installation
# which could prevent them from upgrade, downgrade
# to prevent "conflicting job" error it's not applied
# to "remove" and "reinstall" commands
solver = self._build_comps_solver()
solver._exclude_packages_from_installed_groups(self)
goal.add_protected(self.sack.query().filterm(
name=self.conf.protected_packages))
if not self._run_hawkey_goal(goal, allow_erasing):
if self.conf.debuglevel >= 6:
goal.log_decisions()
msg = dnf.util._format_resolve_problems(goal.problem_rules())
exc = dnf.exceptions.DepsolveError(msg)
else:
self._transaction = self._goal2transaction(goal)
self._ds_callback.end()
timer()
got_transaction = self._transaction is not None and \
len(self._transaction) > 0
if got_transaction:
msg = self._transaction._rpm_limitations()
if msg:
exc = dnf.exceptions.Error(msg)
if exc is not None:
raise exc
self._plugins.run_resolved()
# auto-enable module streams based on installed RPMs
new_pkgs = self._goal.list_installs()
new_pkgs += self._goal.list_upgrades()
new_pkgs += self._goal.list_downgrades()
new_pkgs += self._goal.list_reinstalls()
self.sack.set_modules_enabled_by_pkgset(self._moduleContainer, new_pkgs)
return got_transaction
def do_transaction(self, display=()):
# :api
if not isinstance(display, Sequence):
display = [display]
display = \
[dnf.yum.rpmtrans.LoggingTransactionDisplay()] + list(display)
if not self.transaction:
# packages are not changed, but comps and modules changes need to be committed
self._moduleContainer.save()
self._moduleContainer.updateFailSafeData()
if self._history and (self._history.group or self._history.env):
cmdline = None
if hasattr(self, 'args') and self.args:
cmdline = ' '.join(self.args)
elif hasattr(self, 'cmds') and self.cmds:
cmdline = ' '.join(self.cmds)
old = self.history.last()
if old is None:
rpmdb_version = self.sack._rpmdb_version()
else:
rpmdb_version = old.end_rpmdb_version
self.history.beg(rpmdb_version, [], [], cmdline)
self.history.end(rpmdb_version)
self._plugins.run_pre_transaction()
self._plugins.run_transaction()
self._trans_success = True
return
tid = None
logger.info(_('Running transaction check'))
lock = dnf.lock.build_rpmdb_lock(self.conf.persistdir,
self.conf.exit_on_lock)
with lock:
self.transaction._populate_rpm_ts(self._ts)
msgs = self._run_rpm_check()
if msgs:
msg = _('Error: transaction check vs depsolve:')
logger.error(msg)
for msg in msgs:
logger.error(msg)
raise dnf.exceptions.TransactionCheckError(msg)
logger.info(_('Transaction check succeeded.'))
timer = dnf.logging.Timer('transaction test')
logger.info(_('Running transaction test'))
self._ts.order() # order the transaction
self._ts.clean() # release memory not needed beyond this point
testcb = dnf.yum.rpmtrans.RPMTransaction(self, test=True)
tserrors = self._ts.test(testcb)
if len(tserrors) > 0:
for msg in testcb.messages():
logger.critical(_('RPM: {}').format(msg))
errstring = _('Transaction test error:') + '\n'
for descr in tserrors:
errstring += ' %s\n' % ucd(descr)
summary = self._trans_error_summary(errstring)
if summary:
errstring += '\n' + summary
raise dnf.exceptions.Error(errstring)
del testcb
logger.info(_('Transaction test succeeded.'))
# With RPMTRANS_FLAG_TEST return just before anything is stored permanently
if self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST):
return
timer()
# save module states on disk right before entering rpm transaction,
# because we want system in recoverable state if transaction gets interrupted
self._moduleContainer.save()
self._moduleContainer.updateFailSafeData()
# unset the sigquit handler
timer = dnf.logging.Timer('transaction')
# setup our rpm ts callback
cb = dnf.yum.rpmtrans.RPMTransaction(self, displays=display)
if self.conf.debuglevel < 2:
for display_ in cb.displays:
display_.output = False
self._plugins.run_pre_transaction()
logger.info(_('Running transaction'))
tid = self._run_transaction(cb=cb)
timer()
self._plugins.unload_removed_plugins(self.transaction)
self._plugins.run_transaction()
# log post transaction summary
def _pto_callback(action, tsis):
msgs = []
for tsi in tsis:
msgs.append('{}: {}'.format(action, str(tsi)))
return msgs
for msg in dnf.util._post_transaction_output(self, self.transaction, _pto_callback):
logger.debug(msg)
return tid
def _trans_error_summary(self, errstring):
"""Parse the error string for 'interesting' errors which can
be grouped, such as disk space issues.
:param errstring: the error string
:return: a string containing a summary of the errors
"""
summary = ''
# do disk space report first
p = re.compile(r'needs (\d+)(K|M)B(?: more space)? on the (\S+) filesystem')
disk = {}
for m in p.finditer(errstring):
size_in_mb = int(m.group(1)) if m.group(2) == 'M' else math.ceil(
int(m.group(1)) / 1024.0)
if m.group(3) not in disk:
disk[m.group(3)] = size_in_mb
if disk[m.group(3)] < size_in_mb:
disk[m.group(3)] = size_in_mb
if disk:
summary += _('Disk Requirements:') + "\n"
for k in disk:
summary += " " + P_(
'At least {0}MB more space needed on the {1} filesystem.',
'At least {0}MB more space needed on the {1} filesystem.',
disk[k]).format(disk[k], k) + '\n'
if not summary:
return None
summary = _('Error Summary') + '\n-------------\n' + summary
return summary
def _record_history(self):
return self.conf.history_record and \
not self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST)
def _run_transaction(self, cb):
"""
Perform the RPM transaction.
:return: history database transaction ID or None
"""
tid = None
if self._record_history():
using_pkgs_pats = list(self.conf.history_record_packages)
installed_query = self.sack.query().installed()
using_pkgs = installed_query.filter(name=using_pkgs_pats).run()
rpmdbv = self.sack._rpmdb_version()
lastdbv = self.history.last()
if lastdbv is not None:
lastdbv = lastdbv.end_rpmdb_version
if lastdbv is None or rpmdbv != lastdbv:
logger.debug(_("RPMDB altered outside of {prog}.").format(
prog=dnf.util.MAIN_PROG_UPPER))
cmdline = None
if hasattr(self, 'args') and self.args:
cmdline = ' '.join(self.args)
elif hasattr(self, 'cmds') and self.cmds:
cmdline = ' '.join(self.cmds)
comment = self.conf.comment if self.conf.comment else ""
tid = self.history.beg(rpmdbv, using_pkgs, [], cmdline, comment)
if self.conf.reset_nice:
onice = os.nice(0)
if onice:
try:
os.nice(-onice)
except:
onice = 0
logger.log(dnf.logging.DDEBUG, 'RPM transaction start.')
errors = self._ts.run(cb.callback, '')
logger.log(dnf.logging.DDEBUG, 'RPM transaction over.')
# ts.run() exit codes are, hmm, "creative": None means all ok, empty
# list means some errors happened in the transaction and non-empty
# list that there were errors preventing the ts from starting...
if self.conf.reset_nice:
try:
os.nice(onice)
except:
pass
dnf.util._sync_rpm_trans_with_swdb(self._ts, self._transaction)
if errors is None:
pass
elif len(errors) == 0:
# If there is no failing element it means that some "global" error
# occurred (like rpm failed to obtain the transaction lock). Just pass
# the rpm logs on to the user and raise an Error.
# If there are failing elements the problem is related to those
# elements and the Error is raised later, after saving the failure
# to the history and printing out the transaction table to user.
failed = [el for el in self._ts if el.Failed()]
if not failed:
for msg in cb.messages():
logger.critical(_('RPM: {}').format(msg))
msg = _('Could not run transaction.')
raise dnf.exceptions.Error(msg)
else:
logger.critical(_("Transaction couldn't start:"))
for e in errors:
logger.critical(ucd(e[0]))
if self._record_history() and not self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST):
self.history.end(rpmdbv)
msg = _("Could not run transaction.")
raise dnf.exceptions.Error(msg)
for i in ('ts_all_fn', 'ts_done_fn'):
if hasattr(cb, i):
fn = getattr(cb, i)
try:
misc.unlink_f(fn)
except (IOError, OSError):
msg = _('Failed to remove transaction file %s')
logger.critical(msg, fn)
# keep install_set status because _verify_transaction will clean it
self._trans_install_set = bool(self._transaction.install_set)
# sync up what just happened versus what is in the rpmdb
if not self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST):
self._verify_transaction(cb.verify_tsi_package)
return tid
def _verify_transaction(self, verify_pkg_cb=None):
transaction_items = [
tsi for tsi in self.transaction
if tsi.action != libdnf.transaction.TransactionItemAction_REASON_CHANGE]
total = len(transaction_items)
def display_banner(pkg, count):
count += 1
if verify_pkg_cb is not None:
verify_pkg_cb(pkg, count, total)
return count
timer = dnf.logging.Timer('verify transaction')
count = 0
rpmdb_sack = dnf.sack.rpmdb_sack(self)
# mark group packages that are installed on the system as installed in the db
q = rpmdb_sack.query().installed()
names = set([i.name for i in q])
for ti in self.history.group:
g = ti.getCompsGroupItem()
for p in g.getPackages():
if p.getName() in names:
p.setInstalled(True)
p.save()
# TODO: installed groups in environments
# Post-transaction verification is no longer needed,
# because DNF trusts error codes returned by RPM.
# Verification banner is displayed to preserve UX.
# TODO: drop in future DNF
for tsi in transaction_items:
count = display_banner(tsi.pkg, count)
rpmdbv = rpmdb_sack._rpmdb_version()
self.history.end(rpmdbv)
timer()
self._trans_success = True
def _download_remote_payloads(self, payloads, drpm, progress, callback_total, fail_fast=True):
lock = dnf.lock.build_download_lock(self.conf.cachedir, self.conf.exit_on_lock)
with lock:
beg_download = time.time()
est_remote_size = sum(pload.download_size for pload in payloads)
total_drpm = len(
[payload for payload in payloads if isinstance(payload, dnf.drpm.DeltaPayload)])
# compatibility part for tools that do not accept total_drpms keyword
if progress.start.__code__.co_argcount == 4:
progress.start(len(payloads), est_remote_size, total_drpms=total_drpm)
else:
progress.start(len(payloads), est_remote_size)
errors = dnf.repo._download_payloads(payloads, drpm, fail_fast)
if errors._irrecoverable():
raise dnf.exceptions.DownloadError(errors._irrecoverable())
remote_size = sum(errors._bandwidth_used(pload)
for pload in payloads)
saving = dnf.repo._update_saving((0, 0), payloads,
errors._recoverable)
retries = self.conf.retries
forever = retries == 0
while errors._recoverable and (forever or retries > 0):
if retries > 0:
retries -= 1
msg = _("Some packages were not downloaded. Retrying.")
logger.info(msg)
remaining_pkgs = [pkg for pkg in errors._recoverable]
payloads = \
[dnf.repo._pkg2payload(pkg, progress, dnf.repo.RPMPayload)
for pkg in remaining_pkgs]
est_remote_size = sum(pload.download_size
for pload in payloads)
progress.start(len(payloads), est_remote_size)
errors = dnf.repo._download_payloads(payloads, drpm, fail_fast)
if errors._irrecoverable():
raise dnf.exceptions.DownloadError(errors._irrecoverable())
remote_size += \
sum(errors._bandwidth_used(pload) for pload in payloads)
saving = dnf.repo._update_saving(saving, payloads, {})
if errors._recoverable:
msg = dnf.exceptions.DownloadError.errmap2str(
errors._recoverable)
logger.info(msg)
if callback_total is not None:
callback_total(remote_size, beg_download)
(real, full) = saving
if real != full:
if real < full:
msg = _("Delta RPMs reduced %.1f MB of updates to %.1f MB "
"(%d.1%% saved)")
elif real > full:
msg = _("Failed Delta RPMs increased %.1f MB of updates to %.1f MB "
"(%d.1%% wasted)")
percent = 100 - real / full * 100
logger.info(msg, full / 1024 ** 2, real / 1024 ** 2, percent)
def download_packages(self, pkglist, progress=None, callback_total=None):
# :api
"""Download the packages specified by the given list of packages.
`pkglist` is a list of packages to download, `progress` is an optional
DownloadProgress instance, `callback_total` an optional callback to
output messages about the download operation.
"""
remote_pkgs, local_pkgs = self._select_remote_pkgs(pkglist)
if remote_pkgs:
if progress is None:
progress = dnf.callback.NullDownloadProgress()
drpm = dnf.drpm.DeltaInfo(self.sack.query().installed(),
progress, self.conf.deltarpm_percentage)
self._add_tempfiles([pkg.localPkg() for pkg in remote_pkgs])
payloads = [dnf.repo._pkg2payload(pkg, progress, drpm.delta_factory,
dnf.repo.RPMPayload)
for pkg in remote_pkgs]
self._download_remote_payloads(payloads, drpm, progress, callback_total)
if self.conf.destdir:
for pkg in local_pkgs:
if pkg.baseurl:
location = os.path.join(pkg.get_local_baseurl(),
pkg.location.lstrip("/"))
else:
location = os.path.join(pkg.repo.pkgdir, pkg.location.lstrip("/"))
shutil.copy(location, self.conf.destdir)
def add_remote_rpms(self, path_list, strict=True, progress=None):
# :api
pkgs = []
if not path_list:
return pkgs
if self._goal.req_length():
raise dnf.exceptions.Error(
_("Cannot add local packages, because transaction job already exists"))
pkgs_error = []
for path in path_list:
if not os.path.exists(path) and '://' in path:
# download remote rpm to a tempfile
path = dnf.util._urlopen_progress(path, self.conf, progress)
self._add_tempfiles([path])
try:
pkgs.append(self.sack.add_cmdline_package(path))
except IOError as e:
logger.warning(e)
pkgs_error.append(path)
self._setup_excludes_includes(only_main=True)
if pkgs_error and strict:
raise IOError(_("Could not open: {}").format(' '.join(pkgs_error)))
return pkgs
def _sig_check_pkg(self, po):
"""Verify the GPG signature of the given package object.
:param po: the package object to verify the signature of
:return: (result, error_string)
where result is::
0 = GPG signature verifies ok or verification is not required.
1 = GPG verification failed but installation of the right GPG key
might help.
2 = Fatal GPG verification error, give up.
"""
if po._from_cmdline:
check = self.conf.localpkg_gpgcheck
hasgpgkey = 0
else:
repo = self.repos[po.repoid]
check = repo.gpgcheck
hasgpgkey = not not repo.gpgkey
if check:
root = self.conf.installroot
ts = dnf.rpm.transaction.initReadOnlyTransaction(root)
sigresult = dnf.rpm.miscutils.checkSig(ts, po.localPkg())
localfn = os.path.basename(po.localPkg())
del ts
if sigresult == 0:
result = 0
msg = ''
elif sigresult == 1:
if hasgpgkey:
result = 1
else:
result = 2
msg = _('Public key for %s is not installed') % localfn
elif sigresult == 2:
result = 2
msg = _('Problem opening package %s') % localfn
elif sigresult == 3:
if hasgpgkey:
result = 1
else:
result = 2
result = 1
msg = _('Public key for %s is not trusted') % localfn
elif sigresult == 4:
result = 2
msg = _('Package %s is not signed') % localfn
else:
result = 0
msg = ''
return result, msg
def package_signature_check(self, pkg):
# :api
"""Verify the GPG signature of the given package object.
:param pkg: the package object to verify the signature of
:return: (result, error_string)
where result is::
0 = GPG signature verifies ok or verification is not required.
1 = GPG verification failed but installation of the right GPG key
might help.
2 = Fatal GPG verification error, give up.
"""
return self._sig_check_pkg(pkg)
def _clean_packages(self, packages):
for fn in packages:
if not os.path.exists(fn):
continue
try:
misc.unlink_f(fn)
except OSError:
logger.warning(_('Cannot remove %s'), fn)
continue
else:
logger.log(dnf.logging.DDEBUG,
_('%s removed'), fn)
def _do_package_lists(self, pkgnarrow='all', patterns=None, showdups=None,
ignore_case=False, reponame=None):
"""Return a :class:`misc.GenericHolder` containing
lists of package objects. The contents of the lists are
specified in various ways by the arguments.
:param pkgnarrow: a string specifying which types of packages
lists to produces, such as updates, installed, available,
etc.
:param patterns: a list of names or wildcards specifying
packages to list
:param showdups: whether to include duplicate packages in the
lists
:param ignore_case: whether to ignore case when searching by
package names
:param reponame: limit packages list to the given repository
:return: a :class:`misc.GenericHolder` instance with the
following lists defined::
available = list of packageObjects
installed = list of packageObjects
upgrades = tuples of packageObjects (updating, installed)
extras = list of packageObjects
obsoletes = tuples of packageObjects (obsoleting, installed)
recent = list of packageObjects
"""
if showdups is None:
showdups = self.conf.showdupesfromrepos
if patterns is None:
return self._list_pattern(
pkgnarrow, patterns, showdups, ignore_case, reponame)
assert not dnf.util.is_string_type(patterns)
list_fn = functools.partial(
self._list_pattern, pkgnarrow, showdups=showdups,
ignore_case=ignore_case, reponame=reponame)
if patterns is None or len(patterns) == 0:
return list_fn(None)
yghs = map(list_fn, patterns)
return functools.reduce(lambda a, b: a.merge_lists(b), yghs)
def _list_pattern(self, pkgnarrow, pattern, showdups, ignore_case,
reponame=None):
def is_from_repo(package):
"""Test whether given package originates from the repository."""
if reponame is None:
return True
return self.history.repo(package) == reponame
def pkgs_from_repo(packages):
"""Filter out the packages which do not originate from the repo."""
return (package for package in packages if is_from_repo(package))
def query_for_repo(query):
"""Filter out the packages which do not originate from the repo."""
if reponame is None:
return query
return query.filter(reponame=reponame)
ygh = misc.GenericHolder(iter=pkgnarrow)
installed = []
available = []
reinstall_available = []
old_available = []
updates = []
obsoletes = []
obsoletesTuples = []
recent = []
extras = []
autoremove = []
# do the initial pre-selection
ic = ignore_case
q = self.sack.query()
if pattern is not None:
subj = dnf.subject.Subject(pattern, ignore_case=ic)
q = subj.get_best_query(self.sack, with_provides=False)
# list all packages - those installed and available:
if pkgnarrow == 'all':
dinst = {}
ndinst = {} # Newest versions by name.arch
for po in q.installed():
dinst[po.pkgtup] = po
if showdups:
continue
key = (po.name, po.arch)
if key not in ndinst or po > ndinst[key]:
ndinst[key] = po
installed = list(pkgs_from_repo(dinst.values()))
avail = query_for_repo(q.available())
if not showdups:
avail = avail.filterm(latest_per_arch_by_priority=True)
for pkg in avail:
if showdups:
if pkg.pkgtup in dinst:
reinstall_available.append(pkg)
else:
available.append(pkg)
else:
key = (pkg.name, pkg.arch)
if pkg.pkgtup in dinst:
reinstall_available.append(pkg)
elif key not in ndinst or pkg.evr_gt(ndinst[key]):
available.append(pkg)
else:
old_available.append(pkg)
# produce the updates list of tuples
elif pkgnarrow == 'upgrades':
updates = query_for_repo(q).filterm(upgrades_by_priority=True)
# reduce a query to security upgrades if they are specified
updates = self._merge_update_filters(updates, upgrade=True)
# reduce a query to remove src RPMs
updates.filterm(arch__neq=['src', 'nosrc'])
# reduce a query to latest packages
updates = updates.latest().run()
# installed only
elif pkgnarrow == 'installed':
installed = list(pkgs_from_repo(q.installed()))
# available in a repository
elif pkgnarrow == 'available':
if showdups:
avail = query_for_repo(q).available()
installed_dict = q.installed()._na_dict()
for avail_pkg in avail:
key = (avail_pkg.name, avail_pkg.arch)
installed_pkgs = installed_dict.get(key, [])
same_ver = [pkg for pkg in installed_pkgs
if pkg.evr == avail_pkg.evr]
if len(same_ver) > 0:
reinstall_available.append(avail_pkg)
else:
available.append(avail_pkg)
else:
# we will only look at the latest versions of packages:
available_dict = query_for_repo(
q).available().filterm(latest_per_arch_by_priority=True)._na_dict()
installed_dict = q.installed().latest()._na_dict()
for (name, arch) in available_dict:
avail_pkg = available_dict[(name, arch)][0]
inst_pkg = installed_dict.get((name, arch), [None])[0]
if not inst_pkg or avail_pkg.evr_gt(inst_pkg):
available.append(avail_pkg)
elif avail_pkg.evr_eq(inst_pkg):
reinstall_available.append(avail_pkg)
else:
old_available.append(avail_pkg)
# packages to be removed by autoremove
elif pkgnarrow == 'autoremove':
autoremove_q = query_for_repo(q)._unneeded(self.history.swdb)
autoremove = autoremove_q.run()
# not in a repo but installed
elif pkgnarrow == 'extras':
extras = [pkg for pkg in q.extras() if is_from_repo(pkg)]
# obsoleting packages (and what they obsolete)
elif pkgnarrow == 'obsoletes':
inst = q.installed()
obsoletes = query_for_repo(
self.sack.query()).filter(obsoletes_by_priority=inst)
# reduce a query to security upgrades if they are specified
obsoletes = self._merge_update_filters(obsoletes, warning=False, upgrade=True)
# reduce a query to remove src RPMs
obsoletes.filterm(arch__neq=['src', 'nosrc'])
obsoletesTuples = []
for new in obsoletes:
obsoleted_reldeps = new.obsoletes
obsoletesTuples.extend(
[(new, old) for old in
inst.filter(provides=obsoleted_reldeps)])
# packages recently added to the repositories
elif pkgnarrow == 'recent':
avail = q.available()
if not showdups:
avail = avail.filterm(latest_per_arch_by_priority=True)
recent = query_for_repo(avail)._recent(self.conf.recent)
ygh.installed = installed
ygh.available = available
ygh.reinstall_available = reinstall_available
ygh.old_available = old_available
ygh.updates = updates
ygh.obsoletes = obsoletes
ygh.obsoletesTuples = obsoletesTuples
ygh.recent = recent
ygh.extras = extras
ygh.autoremove = autoremove
return ygh
def _add_comps_trans(self, trans):
self._comps_trans += trans
return len(trans)
def _remove_if_unneeded(self, query):
"""
Mark to remove packages that are not required by any user installed package (reason group
or user)
:param query: dnf.query.Query() object
"""
query = query.installed()
if not query:
return
unneeded_pkgs = query._safe_to_remove(self.history.swdb, debug_solver=False)
unneeded_pkgs_history = query.filter(
pkg=[i for i in query if self.history.group.is_removable_pkg(i.name)])
pkg_with_dependent_pkgs = unneeded_pkgs_history.difference(unneeded_pkgs)
# mark packages with dependent packages as a dependency to allow removal with dependent
# package
for pkg in pkg_with_dependent_pkgs:
self.history.set_reason(pkg, libdnf.transaction.TransactionItemReason_DEPENDENCY)
unneeded_pkgs = unneeded_pkgs.intersection(unneeded_pkgs_history)
remove_packages = query.intersection(unneeded_pkgs)
if remove_packages:
for pkg in remove_packages:
self._goal.erase(pkg, clean_deps=self.conf.clean_requirements_on_remove)
def _finalize_comps_trans(self):
trans = self._comps_trans
basearch = self.conf.substitutions['basearch']
def trans_upgrade(query, remove_query, comps_pkg):
sltr = dnf.selector.Selector(self.sack)
sltr.set(pkg=query)
self._goal.upgrade(select=sltr)
return remove_query
def trans_install(query, remove_query, comps_pkg, strict):
if self.conf.multilib_policy == "all":
if not comps_pkg.requires:
self._install_multiarch(query, strict=strict)
else:
# it installs only one arch for conditional packages
installed_query = query.installed().apply()
self._report_already_installed(installed_query)
sltr = dnf.selector.Selector(self.sack)
sltr.set(provides="({} if {})".format(comps_pkg.name, comps_pkg.requires))
self._goal.install(select=sltr, optional=not strict)
else:
sltr = dnf.selector.Selector(self.sack)
if comps_pkg.requires:
sltr.set(provides="({} if {})".format(comps_pkg.name, comps_pkg.requires))
else:
if self.conf.obsoletes:
query = query.union(self.sack.query().filterm(obsoletes=query))
sltr.set(pkg=query)
self._goal.install(select=sltr, optional=not strict)
return remove_query
def trans_remove(query, remove_query, comps_pkg):
remove_query = remove_query.union(query)
return remove_query
remove_query = self.sack.query().filterm(empty=True)
attr_fn = ((trans.install, functools.partial(trans_install, strict=True)),
(trans.install_opt, functools.partial(trans_install, strict=False)),
(trans.upgrade, trans_upgrade),
(trans.remove, trans_remove))
for (attr, fn) in attr_fn:
for comps_pkg in attr:
query_args = {'name': comps_pkg.name}
if (comps_pkg.basearchonly):
query_args.update({'arch': basearch})
q = self.sack.query().filterm(**query_args).apply()
q.filterm(arch__neq=["src", "nosrc"])
if not q:
package_string = comps_pkg.name
if comps_pkg.basearchonly:
package_string += '.' + basearch
logger.warning(_('No match for group package "{}"').format(package_string))
continue
remove_query = fn(q, remove_query, comps_pkg)
self._goal.group_members.add(comps_pkg.name)
self._remove_if_unneeded(remove_query)
def _build_comps_solver(self):
def reason_fn(pkgname):
q = self.sack.query().installed().filterm(name=pkgname)
if not q:
return None
try:
return self.history.rpm.get_reason(q[0])
except AttributeError:
return libdnf.transaction.TransactionItemReason_UNKNOWN
return dnf.comps.Solver(self.history, self._comps, reason_fn)
def environment_install(self, env_id, types, exclude=None, strict=True, exclude_groups=None):
# :api
"""Installs packages of environment group identified by env_id.
:param types: Types of packages to install. Either an integer as a
logical conjunction of CompsPackageType ids or a list of string
package type ids (conditional, default, mandatory, optional).
"""
assert dnf.util.is_string_type(env_id)
solver = self._build_comps_solver()
if not isinstance(types, int):
types = libdnf.transaction.listToCompsPackageType(types)
trans = solver._environment_install(env_id, types, exclude or set(), strict, exclude_groups)
if not trans:
return 0
return self._add_comps_trans(trans)
def environment_remove(self, env_id):
# :api
assert dnf.util.is_string_type(env_id)
solver = self._build_comps_solver()
trans = solver._environment_remove(env_id)
return self._add_comps_trans(trans)
def group_install(self, grp_id, pkg_types, exclude=None, strict=True):
# :api
"""Installs packages of selected group
:param pkg_types: Types of packages to install. Either an integer as a
logical conjunction of CompsPackageType ids or a list of string
package type ids (conditional, default, mandatory, optional).
:param exclude: list of package name glob patterns
that will be excluded from install set
:param strict: boolean indicating whether group packages that
exist but are non-installable due to e.g. dependency
issues should be skipped (False) or cause transaction to
fail to resolve (True)
"""
def _pattern_to_pkgname(pattern):
if dnf.util.is_glob_pattern(pattern):
q = self.sack.query().filterm(name__glob=pattern)
return map(lambda p: p.name, q)
else:
return (pattern,)
assert dnf.util.is_string_type(grp_id)
exclude_pkgnames = None
if exclude:
nested_excludes = [_pattern_to_pkgname(p) for p in exclude]
exclude_pkgnames = itertools.chain.from_iterable(nested_excludes)
solver = self._build_comps_solver()
if not isinstance(pkg_types, int):
pkg_types = libdnf.transaction.listToCompsPackageType(pkg_types)
trans = solver._group_install(grp_id, pkg_types, exclude_pkgnames, strict)
if not trans:
return 0
if strict:
instlog = trans.install
else:
instlog = trans.install_opt
logger.debug(_("Adding packages from group '%s': %s"),
grp_id, instlog)
return self._add_comps_trans(trans)
def env_group_install(self, patterns, types, strict=True, exclude=None, exclude_groups=None):
q = CompsQuery(self.comps, self.history, CompsQuery.ENVIRONMENTS | CompsQuery.GROUPS,
CompsQuery.AVAILABLE)
cnt = 0
done = True
for pattern in patterns:
try:
res = q.get(pattern)
except dnf.exceptions.CompsError as err:
logger.error(ucd(err))
done = False
continue
for group_id in res.groups:
if not exclude_groups or group_id not in exclude_groups:
cnt += self.group_install(group_id, types, exclude=exclude, strict=strict)
for env_id in res.environments:
cnt += self.environment_install(env_id, types, exclude=exclude, strict=strict,
exclude_groups=exclude_groups)
if not done and strict:
raise dnf.exceptions.Error(_('Nothing to do.'))
return cnt
def group_remove(self, grp_id):
# :api
assert dnf.util.is_string_type(grp_id)
solver = self._build_comps_solver()
trans = solver._group_remove(grp_id)
return self._add_comps_trans(trans)
def env_group_remove(self, patterns):
q = CompsQuery(self.comps, self.history,
CompsQuery.ENVIRONMENTS | CompsQuery.GROUPS,
CompsQuery.INSTALLED)
try:
res = q.get(*patterns)
except dnf.exceptions.CompsError as err:
logger.error("Warning: %s", ucd(err))
raise dnf.exceptions.Error(_('No groups marked for removal.'))
cnt = 0
for env in res.environments:
cnt += self.environment_remove(env)
for grp in res.groups:
cnt += self.group_remove(grp)
return cnt
def env_group_upgrade(self, patterns):
q = CompsQuery(self.comps, self.history,
CompsQuery.GROUPS | CompsQuery.ENVIRONMENTS,
CompsQuery.INSTALLED)
group_upgraded = False
for pattern in patterns:
try:
res = q.get(pattern)
except dnf.exceptions.CompsError as err:
logger.error(ucd(err))
continue
for env in res.environments:
try:
self.environment_upgrade(env)
group_upgraded = True
except dnf.exceptions.CompsError as err:
logger.error(ucd(err))
continue
for grp in res.groups:
try:
self.group_upgrade(grp)
group_upgraded = True
except dnf.exceptions.CompsError as err:
logger.error(ucd(err))
continue
if not group_upgraded:
msg = _('No group marked for upgrade.')
raise dnf.cli.CliError(msg)
def environment_upgrade(self, env_id):
# :api
assert dnf.util.is_string_type(env_id)
solver = self._build_comps_solver()
trans = solver._environment_upgrade(env_id)
return self._add_comps_trans(trans)
def group_upgrade(self, grp_id):
# :api
assert dnf.util.is_string_type(grp_id)
solver = self._build_comps_solver()
trans = solver._group_upgrade(grp_id)
return self._add_comps_trans(trans)
def _gpg_key_check(self):
"""Checks for the presence of GPG keys in the rpmdb.
:return: 0 if there are no GPG keys in the rpmdb, and 1 if
there are keys
"""
gpgkeyschecked = self.conf.cachedir + '/.gpgkeyschecked.yum'
if os.path.exists(gpgkeyschecked):
return 1
installroot = self.conf.installroot
myts = dnf.rpm.transaction.initReadOnlyTransaction(root=installroot)
myts.pushVSFlags(~(rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS))
idx = myts.dbMatch('name', 'gpg-pubkey')
keys = len(idx)
del idx
del myts
if keys == 0:
return 0
else:
mydir = os.path.dirname(gpgkeyschecked)
if not os.path.exists(mydir):
os.makedirs(mydir)
fo = open(gpgkeyschecked, 'w')
fo.close()
del fo
return 1
def _install_multiarch(self, query, reponame=None, strict=True):
already_inst, available = self._query_matches_installed(query)
self._report_already_installed(already_inst)
for packages in available:
sltr = dnf.selector.Selector(self.sack)
q = self.sack.query().filterm(pkg=packages)
if self.conf.obsoletes:
q = q.union(self.sack.query().filterm(obsoletes=q))
sltr = sltr.set(pkg=q)
if reponame is not None:
sltr = sltr.set(reponame=reponame)
self._goal.install(select=sltr, optional=(not strict))
return len(available)
def _categorize_specs(self, install, exclude):
"""
Categorize :param install and :param exclude list into two groups each (packages and groups)
:param install: list of specs, whether packages ('foo') or groups/modules ('@bar')
:param exclude: list of specs, whether packages ('foo') or groups/modules ('@bar')
:return: categorized install and exclude specs (stored in argparse.Namespace class)
To access packages use: specs.pkg_specs,
to access groups use: specs.grp_specs
"""
install_specs = argparse.Namespace()
exclude_specs = argparse.Namespace()
_parse_specs(install_specs, install)
_parse_specs(exclude_specs, exclude)
return install_specs, exclude_specs
def _exclude_package_specs(self, exclude_specs):
glob_excludes = [exclude for exclude in exclude_specs.pkg_specs
if dnf.util.is_glob_pattern(exclude)]
excludes = [exclude for exclude in exclude_specs.pkg_specs
if exclude not in glob_excludes]
exclude_query = self.sack.query().filter(name=excludes)
glob_exclude_query = self.sack.query().filter(name__glob=glob_excludes)
self.sack.add_excludes(exclude_query)
self.sack.add_excludes(glob_exclude_query)
def _expand_groups(self, group_specs):
groups = set()
q = CompsQuery(self.comps, self.history,
CompsQuery.ENVIRONMENTS | CompsQuery.GROUPS,
CompsQuery.AVAILABLE | CompsQuery.INSTALLED)
for pattern in group_specs:
try:
res = q.get(pattern)
except dnf.exceptions.CompsError as err:
logger.error("Warning: Module or %s", ucd(err))
continue
groups.update(res.groups)
groups.update(res.environments)
for environment_id in res.environments:
environment = self.comps._environment_by_id(environment_id)
for group in environment.groups_iter():
groups.add(group.id)
return list(groups)
def _install_groups(self, group_specs, excludes, skipped, strict=True):
for group_spec in group_specs:
try:
types = self.conf.group_package_types
if '/' in group_spec:
split = group_spec.split('/')
group_spec = split[0]
types = split[1].split(',')
self.env_group_install([group_spec], types, strict, excludes.pkg_specs,
excludes.grp_specs)
except dnf.exceptions.Error:
skipped.append("@" + group_spec)
def install_specs(self, install, exclude=None, reponame=None, strict=True, forms=None):
# :api
if exclude is None:
exclude = []
no_match_group_specs = []
error_group_specs = []
no_match_pkg_specs = []
error_pkg_specs = []
install_specs, exclude_specs = self._categorize_specs(install, exclude)
self._exclude_package_specs(exclude_specs)
for spec in install_specs.pkg_specs:
try:
self.install(spec, reponame=reponame, strict=strict, forms=forms)
except dnf.exceptions.MarkingError as e:
logger.error(str(e))
no_match_pkg_specs.append(spec)
no_match_module_specs = []
module_depsolv_errors = ()
if WITH_MODULES and install_specs.grp_specs:
try:
module_base = dnf.module.module_base.ModuleBase(self)
module_base.install(install_specs.grp_specs, strict)
except dnf.exceptions.MarkingErrors as e:
if e.no_match_group_specs:
for e_spec in e.no_match_group_specs:
no_match_module_specs.append(e_spec)
if e.error_group_specs:
for e_spec in e.error_group_specs:
error_group_specs.append("@" + e_spec)
module_depsolv_errors = e.module_depsolv_errors
else:
no_match_module_specs = install_specs.grp_specs
if no_match_module_specs:
exclude_specs.grp_specs = self._expand_groups(exclude_specs.grp_specs)
self._install_groups(no_match_module_specs, exclude_specs, no_match_group_specs, strict)
if no_match_group_specs or error_group_specs or no_match_pkg_specs or error_pkg_specs \
or module_depsolv_errors:
raise dnf.exceptions.MarkingErrors(no_match_group_specs=no_match_group_specs,
error_group_specs=error_group_specs,
no_match_pkg_specs=no_match_pkg_specs,
error_pkg_specs=error_pkg_specs,
module_depsolv_errors=module_depsolv_errors)
def install(self, pkg_spec, reponame=None, strict=True, forms=None):
# :api
"""Mark package(s) given by pkg_spec and reponame for installation."""
subj = dnf.subject.Subject(pkg_spec)
solution = subj.get_best_solution(self.sack, forms=forms, with_src=False)
if self.conf.multilib_policy == "all" or subj._is_arch_specified(solution):
q = solution['query']
if reponame is not None:
q.filterm(reponame=reponame)
if not q:
self._raise_package_not_found_error(pkg_spec, forms, reponame)
return self._install_multiarch(q, reponame=reponame, strict=strict)
elif self.conf.multilib_policy == "best":
sltrs = subj._get_best_selectors(self,
forms=forms,
obsoletes=self.conf.obsoletes,
reponame=reponame,
reports=True,
solution=solution)
if not sltrs:
self._raise_package_not_found_error(pkg_spec, forms, reponame)
for sltr in sltrs:
self._goal.install(select=sltr, optional=(not strict))
return 1
return 0
def package_downgrade(self, pkg, strict=False):
# :api
if pkg._from_system:
msg = 'downgrade_package() for an installed package.'
raise NotImplementedError(msg)
q = self.sack.query().installed().filterm(name=pkg.name, arch=[pkg.arch, "noarch"])
if not q:
msg = _("Package %s not installed, cannot downgrade it.")
logger.warning(msg, pkg.name)
raise dnf.exceptions.MarkingError(_('No match for argument: %s') % pkg.location, pkg.name)
elif sorted(q)[0] > pkg:
sltr = dnf.selector.Selector(self.sack)
sltr.set(pkg=[pkg])
self._goal.install(select=sltr, optional=(not strict))
return 1
else:
msg = _("Package %s of lower version already installed, "
"cannot downgrade it.")
logger.warning(msg, pkg.name)
return 0
def package_install(self, pkg, strict=True):
# :api
q = self.sack.query()._nevra(pkg.name, pkg.evr, pkg.arch)
already_inst, available = self._query_matches_installed(q)
if pkg in already_inst:
self._report_already_installed([pkg])
elif pkg not in itertools.chain.from_iterable(available):
raise dnf.exceptions.PackageNotFoundError(_('No match for argument: %s'), pkg.location)
else:
sltr = dnf.selector.Selector(self.sack)
sltr.set(pkg=[pkg])
self._goal.install(select=sltr, optional=(not strict))
return 1
def package_reinstall(self, pkg):
if self.sack.query().installed().filterm(name=pkg.name, evr=pkg.evr, arch=pkg.arch):
self._goal.install(pkg)
return 1
msg = _("Package %s not installed, cannot reinstall it.")
logger.warning(msg, str(pkg))
raise dnf.exceptions.MarkingError(_('No match for argument: %s') % pkg.location, pkg.name)
def package_remove(self, pkg):
self._goal.erase(pkg)
return 1
def package_upgrade(self, pkg):
# :api
if pkg._from_system:
msg = 'upgrade_package() for an installed package.'
raise NotImplementedError(msg)
if pkg.arch == 'src':
msg = _("File %s is a source package and cannot be updated, ignoring.")
logger.info(msg, pkg.location)
return 0
installed = self.sack.query().installed().apply()
if self.conf.obsoletes and self.sack.query().filterm(pkg=[pkg]).filterm(obsoletes=installed):
sltr = dnf.selector.Selector(self.sack)
sltr.set(pkg=[pkg])
self._goal.upgrade(select=sltr)
return 1
# do not filter by arch if the package is noarch
if pkg.arch == "noarch":
q = installed.filter(name=pkg.name)
else:
q = installed.filter(name=pkg.name, arch=[pkg.arch, "noarch"])
if not q:
msg = _("Package %s not installed, cannot update it.")
logger.warning(msg, pkg.name)
raise dnf.exceptions.MarkingError(
_('No match for argument: %s') % pkg.location, pkg.name)
elif sorted(q)[-1] < pkg:
sltr = dnf.selector.Selector(self.sack)
sltr.set(pkg=[pkg])
self._goal.upgrade(select=sltr)
return 1
else:
msg = _("The same or higher version of %s is already installed, "
"cannot update it.")
logger.warning(msg, pkg.name)
return 0
def _upgrade_internal(self, query, obsoletes, reponame, pkg_spec=None):
installed_all = self.sack.query().installed()
# Add only relevant obsoletes to transaction => installed, upgrades
q = query.intersection(self.sack.query().filterm(name=[pkg.name for pkg in installed_all]))
installed_query = q.installed()
if obsoletes:
obsoletes = self.sack.query().available().filterm(
obsoletes=installed_query.union(q.upgrades()))
# add obsoletes into transaction
query = query.union(obsoletes)
if reponame is not None:
query.filterm(reponame=reponame)
query = self._merge_update_filters(query, pkg_spec=pkg_spec, upgrade=True)
if query:
# Given that we use libsolv's targeted transactions, we need to ensure that the transaction contains both
# the new targeted version and also the current installed version (for the upgraded package). This is
# because if it only contained the new version, libsolv would decide to reinstall the package even if it
# had just a different buildtime or vendor but the same version
# (https://github.com/openSUSE/libsolv/issues/287)
# - In general, the query already contains both the new and installed versions but not always.
# If repository-packages command is used, the installed packages are filtered out because they are from
# the @system repo. We need to add them back in.
# - However we need to add installed versions of just the packages that are being upgraded. We don't want
# to add all installed packages because it could increase the number of solutions for the transaction
# (especially without --best) and since libsolv prefers the smallest possible upgrade it could result
# in no upgrade even if there is one available. This is a problem in general but its critical with
# --security transactions (https://bugzilla.redhat.com/show_bug.cgi?id=2097757)
# - We want to add only the latest versions of installed packages, this is specifically for installonly
# packages. Otherwise if for example kernel-1 and kernel-3 were installed and present in the
# transaction libsolv could decide to install kernel-2 because it is an upgrade for kernel-1 even
# though we don't want it because there already is a newer version present.
query = query.union(installed_all.latest().filter(name=[pkg.name for pkg in query]))
sltr = dnf.selector.Selector(self.sack)
sltr.set(pkg=query)
self._goal.upgrade(select=sltr)
return 1
def upgrade(self, pkg_spec, reponame=None):
# :api
subj = dnf.subject.Subject(pkg_spec)
solution = subj.get_best_solution(self.sack)
q = solution["query"]
if q:
wildcard = dnf.util.is_glob_pattern(pkg_spec)
# wildcard shouldn't print not installed packages
# only solution with nevra.name provide packages with same name
if not wildcard and solution['nevra'] and solution['nevra'].name:
pkg_name = solution['nevra'].name
installed = self.sack.query().installed().apply()
obsoleters = q.filter(obsoletes=installed) \
if self.conf.obsoletes else self.sack.query().filterm(empty=True)
if not obsoleters:
installed_name = installed.filter(name=pkg_name).apply()
if not installed_name:
msg = _('Package %s available, but not installed.')
logger.warning(msg, pkg_name)
raise dnf.exceptions.PackagesNotInstalledError(
_('No match for argument: %s') % pkg_spec, pkg_spec)
elif solution['nevra'].arch and not dnf.util.is_glob_pattern(solution['nevra'].arch):
if not installed_name.filterm(arch=solution['nevra'].arch):
msg = _('Package %s available, but installed for different architecture.')
logger.warning(msg, "{}.{}".format(pkg_name, solution['nevra'].arch))
obsoletes = self.conf.obsoletes and solution['nevra'] \
and solution['nevra'].has_just_name()
return self._upgrade_internal(q, obsoletes, reponame, pkg_spec)
raise dnf.exceptions.MarkingError(_('No match for argument: %s') % pkg_spec, pkg_spec)
def upgrade_all(self, reponame=None):
# :api
# provide only available packages to solver to trigger targeted upgrade
# possibilities will be ignored
# usage of selected packages will unify dnf behavior with other upgrade functions
return self._upgrade_internal(
self.sack.query(), self.conf.obsoletes, reponame, pkg_spec=None)
def distro_sync(self, pkg_spec=None):
if pkg_spec is None:
self._goal.distupgrade_all()
else:
subject = dnf.subject.Subject(pkg_spec)
solution = subject.get_best_solution(self.sack, with_src=False)
solution["query"].filterm(reponame__neq=hawkey.SYSTEM_REPO_NAME)
sltrs = subject._get_best_selectors(self, solution=solution,
obsoletes=self.conf.obsoletes, reports=True)
if not sltrs:
logger.info(_('No package %s installed.'), pkg_spec)
return 0
for sltr in sltrs:
self._goal.distupgrade(select=sltr)
return 1
def autoremove(self, forms=None, pkg_specs=None, grp_specs=None, filenames=None):
# :api
"""Removes all 'leaf' packages from the system that were originally
installed as dependencies of user-installed packages but which are
no longer required by any such package."""
if any([grp_specs, pkg_specs, filenames]):
pkg_specs += filenames
done = False
# Remove groups.
if grp_specs and forms:
for grp_spec in grp_specs:
msg = _('Not a valid form: %s')
logger.warning(msg, grp_spec)
elif grp_specs:
if self.env_group_remove(grp_specs):
done = True
for pkg_spec in pkg_specs:
try:
self.remove(pkg_spec, forms=forms)
except dnf.exceptions.MarkingError as e:
logger.info(str(e))
else:
done = True
if not done:
logger.warning(_('No packages marked for removal.'))
else:
pkgs = self.sack.query()._unneeded(self.history.swdb,
debug_solver=self.conf.debug_solver)
for pkg in pkgs:
self.package_remove(pkg)
def remove(self, pkg_spec, reponame=None, forms=None):
# :api
"""Mark the specified package for removal."""
matches = dnf.subject.Subject(pkg_spec).get_best_query(self.sack, forms=forms)
installed = [
pkg for pkg in matches.installed()
if reponame is None or
self.history.repo(pkg) == reponame]
if not installed:
self._raise_package_not_installed_error(pkg_spec, forms, reponame)
clean_deps = self.conf.clean_requirements_on_remove
for pkg in installed:
self._goal.erase(pkg, clean_deps=clean_deps)
return len(installed)
def reinstall(self, pkg_spec, old_reponame=None, new_reponame=None,
new_reponame_neq=None, remove_na=False):
subj = dnf.subject.Subject(pkg_spec)
q = subj.get_best_query(self.sack)
installed_pkgs = [
pkg for pkg in q.installed()
if old_reponame is None or
self.history.repo(pkg) == old_reponame]
available_q = q.available()
if new_reponame is not None:
available_q.filterm(reponame=new_reponame)
if new_reponame_neq is not None:
available_q.filterm(reponame__neq=new_reponame_neq)
available_nevra2pkg = dnf.query._per_nevra_dict(available_q)
if not installed_pkgs:
raise dnf.exceptions.PackagesNotInstalledError(
'no package matched', pkg_spec, available_nevra2pkg.values())
cnt = 0
clean_deps = self.conf.clean_requirements_on_remove
for installed_pkg in installed_pkgs:
try:
available_pkg = available_nevra2pkg[ucd(installed_pkg)]
except KeyError:
if not remove_na:
continue
self._goal.erase(installed_pkg, clean_deps=clean_deps)
else:
self._goal.install(available_pkg)
cnt += 1
if cnt == 0:
raise dnf.exceptions.PackagesNotAvailableError(
'no package matched', pkg_spec, installed_pkgs)
return cnt
def downgrade(self, pkg_spec):
# :api
"""Mark a package to be downgraded.
This is equivalent to first removing the currently installed package,
and then installing an older version.
"""
return self.downgrade_to(pkg_spec)
def downgrade_to(self, pkg_spec, strict=False):
"""Downgrade to specific version if specified otherwise downgrades
to one version lower than the package installed.
"""
subj = dnf.subject.Subject(pkg_spec)
q = subj.get_best_query(self.sack)
if not q:
msg = _('No match for argument: %s') % pkg_spec
raise dnf.exceptions.PackageNotFoundError(msg, pkg_spec)
done = 0
available_pkgs = q.available()
available_pkg_names = list(available_pkgs._name_dict().keys())
q_installed = self.sack.query().installed().filterm(name=available_pkg_names)
if len(q_installed) == 0:
msg = _('Packages for argument %s available, but not installed.') % pkg_spec
raise dnf.exceptions.PackagesNotInstalledError(msg, pkg_spec, available_pkgs)
for pkg_name in q_installed._name_dict().keys():
downgrade_pkgs = available_pkgs.downgrades().filter(name=pkg_name)
if not downgrade_pkgs:
msg = _("Package %s of lowest version already installed, cannot downgrade it.")
logger.warning(msg, pkg_name)
continue
sltr = dnf.selector.Selector(self.sack)
sltr.set(pkg=downgrade_pkgs)
self._goal.install(select=sltr, optional=(not strict))
done = 1
return done
def provides(self, provides_spec):
providers = self.sack.query().filterm(file__glob=provides_spec)
if providers:
return providers, [provides_spec]
providers = dnf.query._by_provides(self.sack, provides_spec)
if providers:
return providers, [provides_spec]
if provides_spec.startswith('/bin/') or provides_spec.startswith('/sbin/'):
# compatibility for packages that didn't do UsrMove
binary_provides = ['/usr' + provides_spec]
elif provides_spec.startswith('/'):
# provides_spec is a file path
return providers, [provides_spec]
else:
# suppose that provides_spec is a command, search in /usr/sbin/
binary_provides = [prefix + provides_spec
for prefix in ['/bin/', '/sbin/', '/usr/bin/', '/usr/sbin/']]
return self.sack.query().filterm(file__glob=binary_provides), binary_provides
def add_security_filters(self, cmp_type, types=(), advisory=(), bugzilla=(), cves=(), severity=()):
# :api
"""
It modifies results of install, upgrade, and distrosync methods according to provided
filters.
:param cmp_type: only 'eq' or 'gte' allowed
:param types: List or tuple with strings. E.g. 'bugfix', 'enhancement', 'newpackage',
'security'
:param advisory: List or tuple with strings. E.g.Eg. FEDORA-2201-123
:param bugzilla: List or tuple with strings. Include packages that fix a Bugzilla ID,
Eg. 123123.
:param cves: List or tuple with strings. Include packages that fix a CVE
(Common Vulnerabilities and Exposures) ID. Eg. CVE-2201-0123
:param severity: List or tuple with strings. Includes packages that provide a fix
for an issue of the specified severity.
"""
cmp_dict = {'eq': '__eqg', 'gte': '__eqg__gt'}
if cmp_type not in cmp_dict:
raise ValueError("Unsupported value for `cmp_type`")
cmp = cmp_dict[cmp_type]
if types:
key = 'advisory_type' + cmp
self._update_security_options.setdefault(key, set()).update(types)
if advisory:
key = 'advisory' + cmp
self._update_security_options.setdefault(key, set()).update(advisory)
if bugzilla:
key = 'advisory_bug' + cmp
self._update_security_options.setdefault(key, set()).update(bugzilla)
if cves:
key = 'advisory_cve' + cmp
self._update_security_options.setdefault(key, set()).update(cves)
if severity:
key = 'advisory_severity' + cmp
self._update_security_options.setdefault(key, set()).update(severity)
def reset_security_filters(self):
# :api
"""
Reset all security filters
"""
self._update_security_options = {}
def _merge_update_filters(self, q, pkg_spec=None, warning=True, upgrade=False):
"""
Merge Queries in _update_filters and return intersection with q Query
@param q: Query
@return: Query
"""
if not (self._update_security_options or self._update_security_filters) or not q:
return q
merged_queries = self.sack.query().filterm(empty=True)
if self._update_security_filters:
for query in self._update_security_filters:
merged_queries = merged_queries.union(query)
self._update_security_filters = [merged_queries]
if self._update_security_options:
for filter_name, values in self._update_security_options.items():
if upgrade:
filter_name = filter_name + '__upgrade'
kwargs = {filter_name: values}
merged_queries = merged_queries.union(q.filter(**kwargs))
merged_queries = q.intersection(merged_queries)
if not merged_queries:
if warning:
q = q.upgrades()
count = len(q._name_dict().keys())
if count > 0:
if pkg_spec is None:
msg1 = _("No security updates needed, but {} update "
"available").format(count)
msg2 = _("No security updates needed, but {} updates "
"available").format(count)
logger.warning(P_(msg1, msg2, count))
else:
msg1 = _('No security updates needed for "{}", but {} '
'update available').format(pkg_spec, count)
msg2 = _('No security updates needed for "{}", but {} '
'updates available').format(pkg_spec, count)
logger.warning(P_(msg1, msg2, count))
return merged_queries
def _get_key_for_package(self, po, askcb=None, fullaskcb=None):
"""Retrieve a key for a package. If needed, use the given
callback to prompt whether the key should be imported.
:param po: the package object to retrieve the key of
:param askcb: Callback function to use to ask permission to
import a key. The arguments *askcb* should take are the
package object, the userid of the key, and the keyid
:param fullaskcb: Callback function to use to ask permission to
import a key. This differs from *askcb* in that it gets
passed a dictionary so that we can expand the values passed.
:raises: :class:`dnf.exceptions.Error` if there are errors
retrieving the keys
"""
if po._from_cmdline:
# raise an exception, because po.repoid is not in self.repos
msg = _('Unable to retrieve a key for a commandline package: %s')
raise ValueError(msg % po)
repo = self.repos[po.repoid]
key_installed = repo.id in self._repo_set_imported_gpg_keys
keyurls = [] if key_installed else repo.gpgkey
def _prov_key_data(msg):
msg += _('. Failing package is: %s') % (po) + '\n '
msg += _('GPG Keys are configured as: %s') % \
(', '.join(repo.gpgkey))
return msg
user_cb_fail = False
self._repo_set_imported_gpg_keys.add(repo.id)
for keyurl in keyurls:
keys = dnf.crypto.retrieve(keyurl, repo)
for info in keys:
# Check if key is already installed
if misc.keyInstalled(self._ts, info.rpm_id, info.timestamp) >= 0:
msg = _('GPG key at %s (0x%s) is already installed')
logger.info(msg, keyurl, info.short_id)
continue
# DNS Extension: create a key object, pass it to the verification class
# and print its result as an advice to the user.
if self.conf.gpgkey_dns_verification:
dns_input_key = dnf.dnssec.KeyInfo.from_rpm_key_object(info.userid,
info.raw_key)
dns_result = dnf.dnssec.DNSSECKeyVerification.verify(dns_input_key)
logger.info(dnf.dnssec.nice_user_msg(dns_input_key, dns_result))
# Try installing/updating GPG key
info.url = keyurl
if self.conf.gpgkey_dns_verification:
dnf.crypto.log_dns_key_import(info, dns_result)
else:
dnf.crypto.log_key_import(info)
rc = False
if self.conf.assumeno:
rc = False
elif self.conf.assumeyes:
# DNS Extension: We assume, that the key is trusted in case it is valid,
# its existence is explicitly denied or in case the domain is not signed
# and therefore there is no way to know for sure (this is mainly for
# backward compatibility)
# FAQ:
# * What is PROVEN_NONEXISTENCE?
# In DNSSEC, your domain does not need to be signed, but this state
# (not signed) has to be proven by the upper domain. e.g. when example.com.
# is not signed, com. servers have to sign the message, that example.com.
# does not have any signing key (KSK to be more precise).
if self.conf.gpgkey_dns_verification:
if dns_result in (dnf.dnssec.Validity.VALID,
dnf.dnssec.Validity.PROVEN_NONEXISTENCE):
rc = True
logger.info(dnf.dnssec.any_msg(_("The key has been approved.")))
else:
rc = False
logger.info(dnf.dnssec.any_msg(_("The key has been rejected.")))
else:
rc = True
# grab the .sig/.asc for the keyurl, if it exists if it
# does check the signature on the key if it is signed by
# one of our ca-keys for this repo or the global one then
# rc = True else ask as normal.
elif fullaskcb:
rc = fullaskcb({"po": po, "userid": info.userid,
"hexkeyid": info.short_id,
"keyurl": keyurl,
"fingerprint": info.fingerprint,
"timestamp": info.timestamp})
elif askcb:
rc = askcb(po, info.userid, info.short_id)
if not rc:
user_cb_fail = True
continue
# Import the key
# If rpm.RPMTRANS_FLAG_TEST in self._ts, gpg keys cannot be imported successfully
# therefore the flag was removed for import operation
test_flag = self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST)
if test_flag:
orig_flags = self._ts.getTsFlags()
self._ts.setFlags(orig_flags - rpm.RPMTRANS_FLAG_TEST)
result = self._ts.pgpImportPubkey(misc.procgpgkey(info.raw_key))
if test_flag:
self._ts.setFlags(orig_flags)
if result != 0:
msg = _('Key import failed (code %d)') % result
raise dnf.exceptions.Error(_prov_key_data(msg))
logger.info(_('Key imported successfully'))
key_installed = True
if not key_installed and user_cb_fail:
raise dnf.exceptions.Error(_("Didn't install any keys"))
if not key_installed:
msg = _('The GPG keys listed for the "%s" repository are '
'already installed but they are not correct for this '
'package.\n'
'Check that the correct key URLs are configured for '
'this repository.') % repo.name
raise dnf.exceptions.Error(_prov_key_data(msg))
# Check if the newly installed keys helped
result, errmsg = self._sig_check_pkg(po)
if result != 0:
if keyurls:
msg = _("Import of key(s) didn't help, wrong key(s)?")
logger.info(msg)
errmsg = ucd(errmsg)
raise dnf.exceptions.Error(_prov_key_data(errmsg))
def package_import_key(self, pkg, askcb=None, fullaskcb=None):
# :api
"""Retrieve a key for a package. If needed, use the given
callback to prompt whether the key should be imported.
:param pkg: the package object to retrieve the key of
:param askcb: Callback function to use to ask permission to
import a key. The arguments *askcb* should take are the
package object, the userid of the key, and the keyid
:param fullaskcb: Callback function to use to ask permission to
import a key. This differs from *askcb* in that it gets
passed a dictionary so that we can expand the values passed.
:raises: :class:`dnf.exceptions.Error` if there are errors
retrieving the keys
"""
self._get_key_for_package(pkg, askcb, fullaskcb)
def _run_rpm_check(self):
results = []
self._ts.check()
for prob in self._ts.problems():
# Newer rpm (4.8.0+) has problem objects, older have just strings.
# Should probably move to using the new objects, when we can. For
# now just be compatible.
results.append(ucd(prob))
return results
def urlopen(self, url, repo=None, mode='w+b', **kwargs):
# :api
"""
Open the specified absolute url, return a file object
which respects proxy setting even for non-repo downloads
"""
return dnf.util._urlopen(url, self.conf, repo, mode, **kwargs)
def _get_installonly_query(self, q=None):
if q is None:
q = self._sack.query(flags=hawkey.IGNORE_EXCLUDES)
installonly = q.filter(provides=self.conf.installonlypkgs)
return installonly
def _report_icase_hint(self, pkg_spec):
subj = dnf.subject.Subject(pkg_spec, ignore_case=True)
solution = subj.get_best_solution(self.sack, with_nevra=True,
with_provides=False, with_filenames=False)
if solution['query'] and solution['nevra'] and solution['nevra'].name and \
pkg_spec != solution['query'][0].name:
logger.info(_(" * Maybe you meant: {}").format(solution['query'][0].name))
def _select_remote_pkgs(self, install_pkgs):
""" Check checksum of packages from local repositories and returns list packages from remote
repositories that will be downloaded. Packages from commandline are skipped.
:param install_pkgs: list of packages
:return: list of remote pkgs
"""
def _verification_of_packages(pkg_list, logger_msg):
all_packages_verified = True
for pkg in pkg_list:
pkg_successfully_verified = False
try:
pkg_successfully_verified = pkg.verifyLocalPkg()
except Exception as e:
logger.critical(str(e))
if pkg_successfully_verified is not True:
logger.critical(logger_msg.format(pkg, pkg.reponame))
all_packages_verified = False
return all_packages_verified
remote_pkgs = []
local_repository_pkgs = []
for pkg in install_pkgs:
if pkg._is_local_pkg():
if pkg.reponame != hawkey.CMDLINE_REPO_NAME:
local_repository_pkgs.append(pkg)
else:
remote_pkgs.append(pkg)
msg = _('Package "{}" from local repository "{}" has incorrect checksum')
if not _verification_of_packages(local_repository_pkgs, msg):
raise dnf.exceptions.Error(
_("Some packages from local repository have incorrect checksum"))
if self.conf.cacheonly:
msg = _('Package "{}" from repository "{}" has incorrect checksum')
if not _verification_of_packages(remote_pkgs, msg):
raise dnf.exceptions.Error(
_('Some packages have invalid cache, but cannot be downloaded due to '
'"--cacheonly" option'))
remote_pkgs = []
return remote_pkgs, local_repository_pkgs
def _report_already_installed(self, packages):
for pkg in packages:
_msg_installed(pkg)
def _raise_package_not_found_error(self, pkg_spec, forms, reponame):
all_query = self.sack.query(flags=hawkey.IGNORE_EXCLUDES)
subject = dnf.subject.Subject(pkg_spec)
solution = subject.get_best_solution(
self.sack, forms=forms, with_src=False, query=all_query)
if reponame is not None:
solution['query'].filterm(reponame=reponame)
if not solution['query']:
raise dnf.exceptions.PackageNotFoundError(_('No match for argument'), pkg_spec)
else:
with_regular_query = self.sack.query(flags=hawkey.IGNORE_REGULAR_EXCLUDES)
with_regular_query = solution['query'].intersection(with_regular_query)
# Modular filtering is applied on a package set that already has regular excludes
# filtered out. So if a package wasn't filtered out by regular excludes, it must have
# been filtered out by modularity.
if with_regular_query:
msg = _('All matches were filtered out by exclude filtering for argument')
else:
msg = _('All matches were filtered out by modular filtering for argument')
raise dnf.exceptions.PackageNotFoundError(msg, pkg_spec)
def _raise_package_not_installed_error(self, pkg_spec, forms, reponame):
all_query = self.sack.query(flags=hawkey.IGNORE_EXCLUDES).installed()
subject = dnf.subject.Subject(pkg_spec)
solution = subject.get_best_solution(
self.sack, forms=forms, with_src=False, query=all_query)
if not solution['query']:
raise dnf.exceptions.PackagesNotInstalledError(_('No match for argument'), pkg_spec)
if reponame is not None:
installed = [pkg for pkg in solution['query'] if self.history.repo(pkg) == reponame]
else:
installed = solution['query']
if not installed:
msg = _('All matches were installed from a different repository for argument')
else:
msg = _('All matches were filtered out by exclude filtering for argument')
raise dnf.exceptions.PackagesNotInstalledError(msg, pkg_spec)
def setup_loggers(self):
# :api
"""
Setup DNF file loggers based on given configuration file. The loggers are set the same
way as if DNF was run from CLI.
"""
self._logging._setup_from_dnf_conf(self.conf, file_loggers_only=True)
def _skipped_packages(self, report_problems, transaction):
"""returns set of conflicting packages and set of packages with broken dependency that would
be additionally installed when --best and --allowerasing"""
if self._goal.actions & (hawkey.INSTALL | hawkey.UPGRADE | hawkey.UPGRADE_ALL):
best = True
else:
best = False
ng = deepcopy(self._goal)
params = {"allow_uninstall": self._allow_erasing,
"force_best": best,
"ignore_weak": True}
ret = ng.run(**params)
if not ret and report_problems:
msg = dnf.util._format_resolve_problems(ng.problem_rules())
logger.warning(msg)
problem_conflicts = set(ng.problem_conflicts(available=True))
problem_dependency = set(ng.problem_broken_dependency(available=True)) - problem_conflicts
def _nevra(item):
return hawkey.NEVRA(name=item.name, epoch=item.epoch, version=item.version,
release=item.release, arch=item.arch)
# Sometimes, pkg is not in transaction item, therefore, comparing by nevra
transaction_nevras = [_nevra(tsi) for tsi in transaction]
skipped_conflicts = set(
[pkg for pkg in problem_conflicts if _nevra(pkg) not in transaction_nevras])
skipped_dependency = set(
[pkg for pkg in problem_dependency if _nevra(pkg) not in transaction_nevras])
return skipped_conflicts, skipped_dependency
def _msg_installed(pkg):
name = ucd(pkg)
msg = _('Package %s is already installed.')
logger.info(msg, name)
Mini Shell