From: Thomas Lamprecht Date: Wed, 23 Sep 2020 08:01:11 +0000 (+0200) Subject: import 15.2.5 X-Git-Url: https://git.proxmox.com/?p=ceph.git;a=commitdiff_plain;h=f6b5b4d738b87d88d2de35127b6b0e41eae2a272 import 15.2.5 Signed-off-by: Thomas Lamprecht --- diff --git a/ceph/.github/CODEOWNERS b/ceph/.github/CODEOWNERS index 86e37fa11..8eecd7ef5 100644 --- a/ceph/.github/CODEOWNERS +++ b/ceph/.github/CODEOWNERS @@ -7,6 +7,15 @@ /monitoring/prometheus @ceph/dashboard /doc/mgr/dashboard.rst @ceph/dashboard +# Dashboard API team +/src/pybind/mgr/dashboard/controllers @ceph/api +/src/pybind/mgr/dashboard/frontend/src/app/shared/api @ceph/api +/src/pybind/mgr/dashboard/run-backend-api-tests.sh @ceph/api +/qa/suites/rados/dashboard @ceph/api +/qa/tasks/mgr/test_dashboard.py @ceph/api +/qa/tasks/mgr/dashboard @ceph/api +/qa/tasks/mgr/test_module_selftest.py @ceph/api + # For Orchestrator related PRs /src/cephadm @ceph/orchestrators /src/pybind/mgr/orchestrator @ceph/orchestrators diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index bddd18497..6d769090d 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -667,4 +667,4 @@ add_custom_target(tags DEPENDS ctags) find_package(CppCheck) find_package(IWYU) -set(VERSION 15.2.4) +set(VERSION 15.2.5) diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index 547a3fe88..1c7ef2040 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -1,53 +1,15 @@ ->=15.2.4 +>=15.2.5 -------- -* Cephadm: There were a lot of small usability improvements and bug fixes: +* CephFS: Automatic static subtree partitioning policies may now be configured + using the new distributed and random ephemeral pinning extended attributes on + directories. See the documentation for more information: + https://docs.ceph.com/docs/master/cephfs/multimds/ - * Grafana when deployed by Cephadm now binds to all network interfaces. - * ``cephadm check-host`` now prints all detected problems at once. - * Cephadm now calls ``ceph dashboard set-grafana-api-ssl-verify false`` - when generating an SSL certificate for Grafana. - * The Alertmanager is now correctly pointed to the Ceph Dashboard - * ``cephadm adopt`` now supports adopting an Alertmanager - * ``ceph orch ps`` now supports filtering by service name - * ``ceph orch host ls`` now marks hosts as offline, if they are not - accessible. +* Monitors now have a config option ``mon_osd_warn_num_repaired``, 10 by default. + If any OSD has repaired more than this many I/O errors in stored data a + ``OSD_TOO_MANY_REPAIRS`` health warning is generated. -* Cephadm can now deploy NFS Ganesha services. For example, to deploy NFS with - a service id of mynfs, that will use the RADOS pool nfs-ganesha and namespace - nfs-ns:: - - ceph orch apply nfs mynfs nfs-ganesha nfs-ns - -* Cephadm: ``ceph orch ls --export`` now returns all service specifications in - yaml representation that is consumable by ``ceph orch apply``. In addition, - the commands ``orch ps`` and ``orch ls`` now support ``--format yaml`` and - ``--format json-pretty``. - -* Cephadm: ``ceph orch apply osd`` supports a ``--preview`` flag that prints a preview of - the OSD specification before deploying OSDs. This makes it possible to - verify that the specification is correct, before applying it. - -* RGW: The ``radosgw-admin`` sub-commands dealing with orphans -- - ``radosgw-admin orphans find``, ``radosgw-admin orphans find``, - ``radosgw-admin orphans find`` -- have been deprecated. They have - not been actively maintained and they store intermediate results on - the cluster, which could fill a nearly-full cluster. They have been - replaced by a tool, currently considered experimental, - ``rgw-orphan-list``. - -* RBD: The name of the rbd pool object that is used to store - rbd trash purge schedule is changed from "rbd_trash_trash_purge_schedule" - to "rbd_trash_purge_schedule". Users that have already started using - ``rbd trash purge schedule`` functionality and have per pool or namespace - schedules configured should copy "rbd_trash_trash_purge_schedule" - object to "rbd_trash_purge_schedule" before the upgrade and remove - "rbd_trash_purge_schedule" using the following commands in every RBD - pool and namespace where a trash purge schedule was previously - configured:: - - rados -p [-N namespace] cp rbd_trash_trash_purge_schedule rbd_trash_purge_schedule - rados -p [-N namespace] rm rbd_trash_trash_purge_schedule - - or use any other convenient way to restore the schedule after the - upgrade. +* Now when noscrub and/or nodeep-scrub flags are set globally or per pool, + scheduled scrubs of the type disabled will be aborted. All user initiated + scrubs are NOT interrupted. diff --git a/ceph/alpine/APKBUILD b/ceph/alpine/APKBUILD index e62d15946..a15bd142f 100644 --- a/ceph/alpine/APKBUILD +++ b/ceph/alpine/APKBUILD @@ -1,7 +1,7 @@ # Contributor: John Coyle # Maintainer: John Coyle pkgname=ceph -pkgver=15.2.4 +pkgver=15.2.5 pkgrel=0 pkgdesc="Ceph is a distributed object store and file system" pkgusers="ceph" @@ -63,7 +63,7 @@ makedepends=" xmlstarlet yasm " -source="ceph-15.2.4.tar.bz2" +source="ceph-15.2.5.tar.bz2" subpackages=" $pkgname-base $pkgname-common @@ -116,7 +116,7 @@ _sysconfdir=/etc _udevrulesdir=/etc/udev/rules.d _python_sitelib=/usr/lib/python2.7/site-packages -builddir=$srcdir/ceph-15.2.4 +builddir=$srcdir/ceph-15.2.5 build() { export CEPH_BUILD_VIRTUALENV=$builddir diff --git a/ceph/ceph.spec b/ceph/ceph.spec index 74a5f95be..0e12976d0 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -98,7 +98,7 @@ # main package definition ################################################################################# Name: ceph -Version: 15.2.4 +Version: 15.2.5 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -114,7 +114,7 @@ License: LGPL-2.1 and LGPL-3.0 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD- Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: %{?_remote_tarball_prefix}ceph-15.2.4.tar.bz2 +Source0: %{?_remote_tarball_prefix}ceph-15.2.5.tar.bz2 %if 0%{?suse_version} # _insert_obs_source_lines_here ExclusiveArch: x86_64 aarch64 ppc64le s390x @@ -750,6 +750,7 @@ Requires: ceph-base = %{_epoch_prefix}%{version}-%{release} Requires: lvm2 Requires: sudo Requires: libstoragemgmt +Requires: python%{python3_pkgversion}-ceph-common = %{_epoch_prefix}%{version}-%{release} %description osd ceph-osd is the object storage daemon for the Ceph distributed file system. It is responsible for storing objects on a local file system @@ -991,6 +992,12 @@ descriptions, and submitting the command to the appropriate daemon. %package -n python%{python3_pkgversion}-ceph-common Summary: Python 3 utility libraries for Ceph +%if 0%{?fedora} || 0%{?rhel} >= 8 +Requires: python%{python3_pkgversion}-pyyaml +%endif +%if 0%{?suse_version} +Requires: python%{python3_pkgversion}-PyYAML +%endif %if 0%{?suse_version} Group: Development/Libraries/Python %endif @@ -1119,7 +1126,7 @@ This package provides Ceph’s default alerts for Prometheus. # common ################################################################################# %prep -%autosetup -p1 -n ceph-15.2.4 +%autosetup -p1 -n ceph-15.2.5 %build # LTO can be enabled as soon as the following GCC bug is fixed: diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index 244c45170..04122c997 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -750,6 +750,7 @@ Requires: ceph-base = %{_epoch_prefix}%{version}-%{release} Requires: lvm2 Requires: sudo Requires: libstoragemgmt +Requires: python%{python3_pkgversion}-ceph-common = %{_epoch_prefix}%{version}-%{release} %description osd ceph-osd is the object storage daemon for the Ceph distributed file system. It is responsible for storing objects on a local file system @@ -991,6 +992,12 @@ descriptions, and submitting the command to the appropriate daemon. %package -n python%{python3_pkgversion}-ceph-common Summary: Python 3 utility libraries for Ceph +%if 0%{?fedora} || 0%{?rhel} >= 8 +Requires: python%{python3_pkgversion}-pyyaml +%endif +%if 0%{?suse_version} +Requires: python%{python3_pkgversion}-PyYAML +%endif %if 0%{?suse_version} Group: Development/Libraries/Python %endif diff --git a/ceph/changelog.upstream b/ceph/changelog.upstream index 10af93187..242e05bf4 100644 --- a/ceph/changelog.upstream +++ b/ceph/changelog.upstream @@ -1,7 +1,13 @@ -ceph (15.2.4-1bionic) bionic; urgency=medium +ceph (15.2.5-1bionic) bionic; urgency=medium - -- Jenkins Build Slave User Tue, 30 Jun 2020 16:42:44 +0000 + -- Jenkins Build Slave User Tue, 15 Sep 2020 19:09:15 +0000 + +ceph (15.2.5-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Tue, 15 Sep 2020 18:57:01 +0000 ceph (15.2.4-1) stable; urgency=medium diff --git a/ceph/doc/ceph-volume/drive-group.rst b/ceph/doc/ceph-volume/drive-group.rst new file mode 100644 index 000000000..4b2b8ed94 --- /dev/null +++ b/ceph/doc/ceph-volume/drive-group.rst @@ -0,0 +1,12 @@ +.. _ceph-volume-drive-group: + +``drive-group`` +=============== +The drive-group subcommand allows for passing :ref:'drivegroups' specifications +straight to ceph-volume as json. ceph-volume will then attempt to deploy this +drive groups via the batch subcommand. + +The specification can be passed via a file, string argument or on stdin. +See the subcommand help for further details:: + + # ceph-volume drive-group --help diff --git a/ceph/doc/ceph-volume/index.rst b/ceph/doc/ceph-volume/index.rst index 36ce9cc89..a9c18abb7 100644 --- a/ceph/doc/ceph-volume/index.rst +++ b/ceph/doc/ceph-volume/index.rst @@ -65,6 +65,7 @@ and ``ceph-disk`` is fully disabled. Encryption is fully supported. intro systemd inventory + drive-group lvm/index lvm/activate lvm/batch diff --git a/ceph/doc/cephadm/adoption.rst b/ceph/doc/cephadm/adoption.rst index 9444d5670..701c3d75f 100644 --- a/ceph/doc/cephadm/adoption.rst +++ b/ceph/doc/cephadm/adoption.rst @@ -81,6 +81,12 @@ Adoption process # ssh-copy-id -f -i ceph.pub root@ + .. note:: + It is also possible to import an existing ssh key. See + :ref:`ssh errors ` in the troubleshooting + document for instructions describing how to import existing + ssh keys. + #. Tell cephadm which hosts to manage:: # ceph orch host add [ip-address] @@ -105,9 +111,9 @@ Adoption process #. Redeploy MDS daemons by telling cephadm how many daemons to run for each file system. You can list file systems by name with ``ceph fs - ls``. For each file system:: + ls``. Run the following command on the master nodes:: - # ceph orch apply mds + # ceph orch apply mds [--placement=] For example, in a cluster with a single file system called `foo`:: @@ -127,7 +133,7 @@ Adoption process #. Redeploy RGW daemons. Cephadm manages RGW daemons by zone. For each zone, deploy new RGW daemons with cephadm:: - # ceph orch apply rgw [--port ] [--ssl] + # ceph orch apply rgw [--subcluster=] [--port=] [--ssl] [--placement=] where ** can be a simple daemon count, or a list of specific hosts (see :ref:`orchestrator-cli-placement-spec`). diff --git a/ceph/doc/cephadm/concepts.rst b/ceph/doc/cephadm/concepts.rst new file mode 100644 index 000000000..8b1743799 --- /dev/null +++ b/ceph/doc/cephadm/concepts.rst @@ -0,0 +1,120 @@ +Cephadm Concepts +================ + +.. _cephadm-fqdn: + +Fully qualified domain names vs bare host names +----------------------------------------------- + +cephadm has very minimal requirements when it comes to resolving host +names etc. When cephadm initiates an ssh connection to a remote host, +the host name can be resolved in four different ways: + +- a custom ssh config resolving the name to an IP +- via an externally maintained ``/etc/hosts`` +- via explicitly providing an IP address to cephadm: ``ceph orch host add `` +- automatic name resolution via DNS. + +Ceph itself uses the command ``hostname`` to determine the name of the +current host. + +.. note:: + + cephadm demands that the name of the host given via ``ceph orch host add`` + equals the output of ``hostname`` on remote hosts. + +Otherwise cephadm can't be sure, the host names returned by +``ceph * metadata`` match the hosts known to cephadm. This might result +in a :ref:`cephadm-stray-host` warning. + +When configuring new hosts, there are two **valid** ways to set the +``hostname`` of a host: + +1. Using the bare host name. In this case: + +- ``hostname`` returns the bare host name. +- ``hostname -f`` returns the FQDN. + +2. Using the fully qualified domain name as the host name. In this case: + +- ``hostname`` returns the FQDN +- ``hostname -s`` return the bare host name + +Note that ``man hostname`` recommends ``hostname`` to return the bare +host name: + + The FQDN (Fully Qualified Domain Name) of the system is the + name that the resolver(3) returns for the host name, such as, + ursula.example.com. It is usually the hostname followed by the DNS + domain name (the part after the first dot). You can check the FQDN + using ``hostname --fqdn`` or the domain name using ``dnsdomainname``. + + :: + + You cannot change the FQDN with hostname or dnsdomainname. + + The recommended method of setting the FQDN is to make the hostname + be an alias for the fully qualified name using /etc/hosts, DNS, or + NIS. For example, if the hostname was "ursula", one might have + a line in /etc/hosts which reads + + 127.0.1.1 ursula.example.com ursula + +Which means, ``man hostname`` recommends ``hostname`` to return the bare +host name. This in turn means that Ceph will return the bare host names +when executing ``ceph * metadata``. This in turn means cephadm also +requires the bare host name when adding a host to the cluster: +``ceph orch host add ``. + +.. + TODO: This chapter needs to provide way for users to configure + Grafana in the dashboard, as this is right no very hard to do. + +Cephadm Scheduler +----------------- + +Cephadm uses a declarative state to define the layout of the cluster. This +state consists of a list of service specifications containing placement +specifications (See :ref:`orchestrator-cli-service-spec` ). + +Cephadm constantly compares list of actually running daemons in the cluster +with the desired service specifications and will either add or remove new +daemons. + +First, cephadm will select a list of candidate hosts. It first looks for +explicit host names and will select those. In case there are no explicit hosts +defined, cephadm looks for a label specification. If there is no label defined +in the specification, cephadm will select hosts based on a host pattern. If +there is no pattern defined, cepham will finally select all known hosts as +candidates. + +Then, cephadm will consider existing daemons of this services and will try to +avoid moving any daemons. + +Cephadm supports the deployment of a specific amount of services. Let's +consider a service specification like so: + +.. code-block:: yaml + + service_type: mds + service_name: myfs + placement: + count: 3 + label: myfs + +This instructs cephadm to deploy three daemons on hosts labeled with +``myfs`` across the cluster. + +Then, in case there are less than three daemons deployed on the candidate +hosts, cephadm will then then randomly choose hosts for deploying new daemons. + +In case there are more than three daemons deployed, cephadm will remove +existing daemons. + +Finally, cephadm will remove daemons on hosts that are outside of the list of +candidate hosts. + +However, there is a special cases that cephadm needs to consider. + +In case the are fewer hosts selected by the placement specification than +demanded by ``count``, cephadm will only deploy on selected hosts. \ No newline at end of file diff --git a/ceph/doc/cephadm/drivegroups.rst b/ceph/doc/cephadm/drivegroups.rst index ae9b9205f..f1dd523e2 100644 --- a/ceph/doc/cephadm/drivegroups.rst +++ b/ceph/doc/cephadm/drivegroups.rst @@ -40,10 +40,27 @@ This will go out on all the matching hosts and deploy these OSDs. Since we want to have more complex setups, there are more filters than just the 'all' filter. +Also, there is a `--dry-run` flag that can be passed to the `apply osd` command, which gives you a synopsis +of the proposed layout. + +Example:: + + [monitor 1] # ceph orch apply osd -i /path/to/osd_spec.yml --dry-run + + Filters ======= +.. note:: + Filters are applied using a `AND` gate by default. This essentially means that a drive needs to fulfill all filter + criteria in order to get selected. + If you wish to change this behavior you can adjust this behavior by setting + + `filter_logic: OR` # valid arguments are `AND`, `OR` + + in the OSD Specification. + You can assign disks to certain groups by their attributes using filters. The attributes are based off of ceph-volume's disk query. You can retrieve the information diff --git a/ceph/doc/cephadm/index.rst b/ceph/doc/cephadm/index.rst index 4ffcab80a..3156721df 100644 --- a/ceph/doc/cephadm/index.rst +++ b/ceph/doc/cephadm/index.rst @@ -37,3 +37,4 @@ versions of Ceph. Client Setup DriveGroups troubleshooting + concepts \ No newline at end of file diff --git a/ceph/doc/cephadm/install.rst b/ceph/doc/cephadm/install.rst index 4459a7092..e486231c6 100644 --- a/ceph/doc/cephadm/install.rst +++ b/ceph/doc/cephadm/install.rst @@ -105,6 +105,19 @@ or run ``cephadm bootstrap -h`` to see all available options: cluster by putting them in a standard ini-style configuration file and using the ``--config **`` option. +* You can choose the ssh user cephadm will use to connect to hosts by + using the ``--ssh-user **`` option. The ssh key will be added + to ``/home/**/.ssh/authorized_keys``. This user will require + passwordless sudo access. + +* If you are using a container on an authenticated registry that requires + login you may add the three arguments ``--registry-url ``, + ``--registry-username ``, + ``--registry-password `` OR + ``--registry-json ``. Cephadm will attempt + to login to this registry so it may pull your container and then store + the login info in its config database so other hosts added to the cluster + may also make use of the authenticated registry. Enable Ceph CLI =============== @@ -125,9 +138,9 @@ command. There are several ways to do this: # cephadm shell -* It may be helpful to create an alias:: +* To execute ``ceph`` commands, you can also run commands like so:: - # alias ceph='cephadm shell -- ceph' + # cephadm shell -- ceph -s * You can install the ``ceph-common`` package, which contains all of the ceph commands, including ``ceph``, ``rbd``, ``mount.ceph`` (for mounting @@ -257,6 +270,49 @@ hosts to the cluster. No further steps are necessary. # ceph orch daemon add mon newhost1:10.1.2.123 # ceph orch daemon add mon newhost2:10.1.2.0/24 + .. note:: + The **apply** command can be confusing. For this reason, we recommend using + YAML specifications. + + Each 'ceph orch apply mon' command supersedes the one before it. + This means that you must use the proper comma-separated list-based + syntax when you want to apply monitors to more than one host. + If you do not use the proper syntax, you will clobber your work + as you go. + + For example:: + + # ceph orch apply mon host1 + # ceph orch apply mon host2 + # ceph orch apply mon host3 + + This results in only one host having a monitor applied to it: host 3. + + (The first command creates a monitor on host1. Then the second command + clobbers the monitor on host1 and creates a monitor on host2. Then the + third command clobbers the monitor on host2 and creates a monitor on + host3. In this scenario, at this point, there is a monitor ONLY on + host3.) + + To make certain that a monitor is applied to each of these three hosts, + run a command like this:: + + # ceph orch apply mon "host1,host2,host3" + + Instead of using the "ceph orch apply mon" commands, run a command like + this:: + + # ceph orch apply -i file.yaml + + Here is a sample **file.yaml** file:: + + service_type: mon + placement: + hosts: + - host1 + - host2 + - host3 + Deploy OSDs =========== @@ -326,29 +382,26 @@ that configuration isn't already in place (usually in the daemons will start up with default settings (e.g., binding to port 80). -If a realm has not been created yet, first create a realm:: +To deploy a set of radosgw daemons for a particular realm and zone:: - # radosgw-admin realm create --rgw-realm= --default + # ceph orch apply rgw ** ** --placement="** [** ...]" -Next create a new zonegroup:: +For example, to deploy 2 rgw daemons serving the *myorg* realm and the *us-east-1* +zone on *myhost1* and *myhost2*:: - # radosgw-admin zonegroup create --rgw-zonegroup= --master --default + # ceph orch apply rgw myorg us-east-1 --placement="2 myhost1 myhost2" -Next create a zone:: +Cephadm will wait for a healthy cluster and automatically create the supplied realm and zone if they do not exist before deploying the rgw daemon(s) - # radosgw-admin zone create --rgw-zonegroup= --rgw-zone= --master --default +Alternatively, the realm, zonegroup, and zone can be manually created using ``radosgw-admin`` commands:: -To deploy a set of radosgw daemons for a particular realm and zone:: + # radosgw-admin realm create --rgw-realm= --default - # ceph orch apply rgw ** ** --placement="** [** ...]" + # radosgw-admin zonegroup create --rgw-zonegroup= --master --default -For example, to deploy 2 rgw daemons serving the *myorg* realm and the *us-east-1* -zone on *myhost1* and *myhost2*:: + # radosgw-admin zone create --rgw-zonegroup= --rgw-zone= --master --default - # radosgw-admin realm create --rgw-realm=myorg --default - # radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - # radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=us-east-1 --master --default - # ceph orch apply rgw myorg us-east-1 --placement="2 myhost1 myhost2" + # radosgw-admin period update --rgw-realm= --commit See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. @@ -367,8 +420,11 @@ RADOS pool *nfs-ganesha* and namespace *nfs-ns*,:: # ceph orch apply nfs foo nfs-ganesha nfs-ns +.. note:: + Create the *nfs-ganesha* pool first if it doesn't exist. + See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. Deploying custom containers =========================== -It is also possible to choose different containers than the default containers to deploy Ceph. See :ref:`containers` for information about your options in this regard. +It is also possible to choose different containers than the default containers to deploy Ceph. See :ref:`containers` for information about your options in this regard. diff --git a/ceph/doc/cephadm/operations.rst b/ceph/doc/cephadm/operations.rst index 7768b1ff4..198286a3a 100644 --- a/ceph/doc/cephadm/operations.rst +++ b/ceph/doc/cephadm/operations.rst @@ -141,6 +141,21 @@ You will then need to restart the mgr daemon to reload the configuration with:: ceph mgr fail +Configuring a different SSH user +---------------------------------- + +Cephadm must be able to log into all the Ceph cluster nodes as an user +that has enough privileges to download container images, start containers +and execute commands without prompting for a password. If you do not want +to use the "root" user (default option in cephadm), you must provide +cephadm the name of the user that is going to be used to perform all the +cephadm operations. Use the command:: + + ceph cephadm set-user + +Prior to running this the cluster ssh key needs to be added to this users +authorized_keys file and non-root users must have passwordless sudo access. + Customizing the SSH configuration --------------------------------- @@ -193,6 +208,8 @@ Resume cephadm work with:: ceph orch resume +.. _cephadm-stray-host: + CEPHADM_STRAY_HOST ------------------ @@ -216,6 +233,9 @@ You can also disable this warning entirely with:: ceph config set mgr mgr/cephadm/warn_on_stray_hosts false +See :ref:`cephadm-fqdn` for more information about host names and +domain names. + CEPHADM_STRAY_DAEMON -------------------- diff --git a/ceph/doc/cephadm/troubleshooting.rst b/ceph/doc/cephadm/troubleshooting.rst index 29e36958e..a439b3d7d 100644 --- a/ceph/doc/cephadm/troubleshooting.rst +++ b/ceph/doc/cephadm/troubleshooting.rst @@ -103,6 +103,7 @@ Cephadm writes small wrappers that run a containers. Refer to ``/var/lib/ceph///unit.run`` for the container execution command. +.. _cephadm-ssh-errors: ssh errors ---------- @@ -138,7 +139,6 @@ Things users can do: [root@mon1 ~]# ssh -F config -i key root@mon1 -4. There is a limitation right now: the ssh user is always `root`. diff --git a/ceph/doc/cephfs/fs-nfs-exports.rst b/ceph/doc/cephfs/fs-nfs-exports.rst new file mode 100644 index 000000000..9740404bd --- /dev/null +++ b/ceph/doc/cephfs/fs-nfs-exports.rst @@ -0,0 +1,155 @@ +======================= +CephFS Exports over NFS +======================= + +CephFS namespaces can be exported over NFS protocol using the +`NFS-Ganesha NFS server `_. + +Requirements +============ + +- Latest Ceph file system with mgr enabled +- 'nfs-ganesha', 'nfs-ganesha-ceph', 'nfs-ganesha-rados-grace' and + 'nfs-ganesha-rados-urls' packages (version 3.3 and above) + +Create NFS Ganesha Cluster +========================== + +.. code:: bash + + $ ceph nfs cluster create [] + +This creates a common recovery pool for all Ganesha daemons, new user based on +cluster_id and common ganesha config rados object. + +Here type is export type and placement specifies the size of cluster and hosts. +For more details on placement specification refer the `orchestrator doc +`_. +Currently only CephFS export type is supported. + +Update NFS Ganesha Cluster +========================== + +.. code:: bash + + $ ceph nfs cluster update + +This updates the deployed cluster according to the placement value. + +Delete NFS Ganesha Cluster +========================== + +.. code:: bash + + $ ceph nfs cluster delete + +This deletes the deployed cluster. + +List NFS Ganesha Cluster +======================== + +.. code:: bash + + $ ceph nfs cluster ls + +This lists deployed clusters. + +Show NFS Ganesha Cluster Information +==================================== + +.. code:: bash + + $ ceph nfs cluster info [] + +This displays ip and port of deployed cluster. + +Set Customized Ganesha Configuration +==================================== + +.. code:: bash + + $ ceph nfs cluster config set -i + +With this the nfs cluster will use the specified config and it will have +precedence over default config blocks. + +Reset Ganesha Configuration +=========================== + +.. code:: bash + + $ ceph nfs cluster config reset + +This removes the user defined configuration. + +Create CephFS Export +==================== + +.. code:: bash + + $ ceph nfs export create cephfs [--readonly] [--path=/path/in/cephfs] + +It creates export rados objects containing the export block. Here binding is +the pseudo root name and type is export type. + +Delete CephFS Export +==================== + +.. code:: bash + + $ ceph nfs export delete + +It deletes an export in cluster based on pseudo root name (binding). + +List CephFS Export +================== + +.. code:: bash + + $ ceph nfs export ls [--detailed] + +It lists export for a cluster. With detailed option enabled it shows entire +export block. + +Get CephFS Export +================= + +.. code:: bash + + $ ceph nfs export get + +It displays export block for a cluster based on pseudo root name (binding). + +Configuring NFS-Ganesha to export CephFS with vstart +==================================================== + +1) Using cephadm + + .. code:: bash + + $ MDS=1 MON=1 OSD=3 NFS=1 ../src/vstart.sh -n -d --cephadm + + It can deploy only single ganesha daemon with vstart on default ganesha port. + +2) Using test orchestrator + + .. code:: bash + + $ MDS=1 MON=1 OSD=3 NFS=1 ../src/vstart.sh -n -d + + It can deploy multiple ganesha daemons on random port. But this requires + ganesha packages to be installed. + +NFS: It is the number of NFS-Ganesha clusters to be created. + +Mount +===== + +After the exports are successfully created and Ganesha daemons are no longer in +grace period. The exports can be mounted by + +.. code:: bash + + $ mount -t nfs -o port= : + +.. note:: Only NFS v4.0+ is supported. diff --git a/ceph/doc/cephfs/fs-volumes.rst b/ceph/doc/cephfs/fs-volumes.rst index f77bd6c88..4efe26d8b 100644 --- a/ceph/doc/cephfs/fs-volumes.rst +++ b/ceph/doc/cephfs/fs-volumes.rst @@ -174,6 +174,13 @@ The output format is json and contains fields as follows. * path: absolute path of a subvolume * type: subvolume type indicating whether it's clone or subvolume * pool_namespace: RADOS namespace of the subvolume +* features: features supported by the subvolume + +The subvolume "features" are based on the internal version of the subvolume and is a list containing +a subset of the following features, + +* "snapshot-clone": supports cloning using a subvolumes snapshot as the source +* "snapshot-autoprotect": supports automatically protecting snapshots, that are active clone sources, from deletion List subvolumes using:: @@ -204,7 +211,6 @@ The output format is json and contains fields as follows. * created_at: time of creation of snapshot in the format "YYYY-MM-DD HH:MM:SS:ffffff" * data_pool: data pool the snapshot belongs to * has_pending_clones: "yes" if snapshot clone is in progress otherwise "no" -* protected: "yes" if snapshot is protected otherwise "no" * size: snapshot size in bytes Cloning Snapshots @@ -214,10 +220,20 @@ Subvolumes can be created by cloning subvolume snapshots. Cloning is an asynchro data from a snapshot to a subvolume. Due to this bulk copy nature, cloning is currently inefficient for very huge data sets. -Before starting a clone operation, the snapshot should be protected. Protecting a snapshot ensures that the snapshot -cannot be deleted when a clone operation is in progress. Snapshots can be protected using:: +.. note:: Removing a snapshot (source subvolume) would fail if there are pending or in progress clone operations. + +Protecting snapshots prior to cloning was a pre-requisite in the Nautilus release, and the commands to protect/unprotect +snapshots were introduced for this purpose. This pre-requisite, and hence the commands to protect/unprotect, is being +deprecated in mainline CephFS, and may be removed from a future release. + +The commands being deprecated are:: $ ceph fs subvolume snapshot protect [--group_name ] + $ ceph fs subvolume snapshot unprotect [--group_name ] + +.. note:: Using the above commands would not result in an error, but they serve no useful function. + +.. note:: Use subvolume info command to fetch subvolume metadata regarding supported "features" to help decide if protect/unprotect of snapshots is required, based on the "snapshot-autoprotect" feature availability. To initiate a clone operation use:: @@ -243,12 +259,11 @@ A clone can be in one of the following states: #. `pending` : Clone operation has not started #. `in-progress` : Clone operation is in progress -#. `complete` : Clone operation has sucessfully finished +#. `complete` : Clone operation has successfully finished #. `failed` : Clone operation has failed Sample output from an `in-progress` clone operation:: - $ ceph fs subvolume snapshot protect cephfs subvol1 snap1 $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1 $ ceph fs clone status cephfs clone1 { @@ -266,7 +281,7 @@ Sample output from an `in-progress` clone operation:: .. note:: Cloned subvolumes are accessible only after the clone operation has successfully completed. -For a successsful clone operation, `clone status` would look like so:: +For a successful clone operation, `clone status` would look like so:: $ ceph fs clone status cephfs clone1 { @@ -282,14 +297,6 @@ To delete a partial clone use:: $ ceph fs subvolume rm [--group_name ] --force -When no clone operations are in progress or scheduled, the snaphot can be unprotected. To unprotect a snapshot use:: - - $ ceph fs subvolume snapshot unprotect [--group_name ] - -Note that unprotecting a snapshot would fail if there are pending or in progress clone operations. Also note that, -only unprotected snapshots can be removed. This guarantees that a snapshot cannot be deleted when clones are pending -(or in progress). - .. note:: Cloning only synchronizes directories, regular files and symbolic links. Also, inode timestamps (access and modification times) are synchronized upto seconds granularity. @@ -299,7 +306,6 @@ An `in-progress` or a `pending` clone operation can be canceled. To cancel a clo On successful cancelation, the cloned subvolume is moved to `canceled` state:: - $ ceph fs subvolume snapshot protect cephfs subvol1 snap1 $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1 $ ceph fs clone cancel cephfs clone1 $ ceph fs clone status cephfs clone1 diff --git a/ceph/doc/cephfs/index.rst b/ceph/doc/cephfs/index.rst index 4031e1032..58839a3f1 100644 --- a/ceph/doc/cephfs/index.rst +++ b/ceph/doc/cephfs/index.rst @@ -83,6 +83,7 @@ Administration MDS Configuration Settings Manual: ceph-mds <../../man/8/ceph-mds> Export over NFS + Export over NFS with volume nfs interface Application best practices FS volume and subvolumes CephFS Quotas diff --git a/ceph/doc/cephfs/multimds.rst b/ceph/doc/cephfs/multimds.rst index fd49e612f..c2367e891 100644 --- a/ceph/doc/cephfs/multimds.rst +++ b/ceph/doc/cephfs/multimds.rst @@ -135,3 +135,102 @@ directory's export pin. For example: setfattr -n ceph.dir.pin -v 0 a/b # a/b is now pinned to rank 0 and a/ and the rest of its children are still pinned to rank 1 + +Setting subtree partitioning policies +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is also possible to setup **automatic** static partitioning of subtrees via +a set of **policies**. In CephFS, this automatic static partitioning is +referred to as **ephemeral pinning**. Any directory (inode) which is +ephemerally pinned will be automatically assigned to a particular rank +according to a consistent hash of its inode number. The set of all +ephemerally pinned directories should be uniformly distributed across all +ranks. + +Ephemerally pinned directories are so named because the pin may not persist +once the directory inode is dropped from cache. However, an MDS failover does +not affect the ephemeral nature of the pinned directory. The MDS records what +subtrees are ephemerally pinned in its journal so MDS failovers do not drop +this information. + +A directory is either ephemerally pinned or not. Which rank it is pinned to is +derived from its inode number and a consistent hash. This means that +ephemerally pinned directories are somewhat evenly spread across the MDS +cluster. The **consistent hash** also minimizes redistribution when the MDS +cluster grows or shrinks. So, growing an MDS cluster may automatically increase +your metadata throughput with no other administrative intervention. + +Presently, there are two types of ephemeral pinning: + +**Distributed Ephemeral Pins**: This policy indicates that **all** of a +directory's immediate children should be ephemerally pinned. The canonical +example would be the ``/home`` directory: we want every user's home directory +to be spread across the entire MDS cluster. This can be set via: + +:: + + setfattr -n ceph.dir.pin.distributed -v 1 /cephfs/home + + +**Random Ephemeral Pins**: This policy indicates any descendent sub-directory +may be ephemerally pinned. This is set through the extended attribute +``ceph.dir.pin.random`` with the value set to the percentage of directories +that should be pinned. For example: + +:: + + setfattr -n ceph.dir.pin.random -v 0.5 /cephfs/tmp + +Would cause any directory loaded into cache or created under ``/tmp`` to be +ephemerally pinned 50 percent of the time. + +It is recomended to only set this to small values, like ``.001`` or ``0.1%``. +Having too many subtrees may degrade performance. For this reason, the config +``mds_export_ephemeral_random_max`` enforces a cap on the maximum of this +percentage (default: ``.01``). The MDS returns ``EINVAL`` when attempting to +set a value beyond this config. + +Both random and distributed ephemeral pin policies are off by default in +Octopus. The features may be enabled via the +``mds_export_ephemeral_random`` and ``mds_export_ephemeral_distributed`` +configuration options. + +Ephemeral pins may override parent export pins and vice versa. What determines +which policy is followed is the rule of the closest parent: if a closer parent +directory has a conflicting policy, use that one instead. For example: + +:: + + mkdir -p foo/bar1/baz foo/bar2 + setfattr -n ceph.dir.pin -v 0 foo + setfattr -n ceph.dir.pin.distributed -v 1 foo/bar1 + +The ``foo/bar1/baz`` directory will be ephemerally pinned because the +``foo/bar1`` policy overrides the export pin on ``foo``. The ``foo/bar2`` +directory will obey the pin on ``foo`` normally. + +For the reverse situation: + +:: + + mkdir -p home/{patrick,john} + setfattr -n ceph.dir.pin.distributed -v 1 home + setfattr -n ceph.dir.pin -v 2 home/patrick + +The ``home/patrick`` directory and its children will be pinned to rank 2 +because its export pin overrides the policy on ``home``. + +If a directory has an export pin and an ephemeral pin policy, the export pin +applies to the directory itself and the policy to its children. So: + +:: + + mkdir -p home/{patrick,john} + setfattr -n ceph.dir.pin -v 0 home + setfattr -n ceph.dir.pin.distributed -v 1 home + +The home directory inode (and all of its directory fragments) will always be +located on rank 0. All children including ``home/patrick`` and ``home/john`` +will be ephemerally pinned according to the distributed policy. This may only +matter for some obscure performance advantages. All the same, it's mentioned +here so the override policy is clear. diff --git a/ceph/doc/dev/cephadm.rst b/ceph/doc/dev/cephadm.rst index 1784de422..4cd811f78 100644 --- a/ceph/doc/dev/cephadm.rst +++ b/ceph/doc/dev/cephadm.rst @@ -25,11 +25,12 @@ cephadm/cephadm script into memory.) MON=1 MGR=1 OSD=0 MDS=0 ../src/vstart.sh -d -n -x --cephadm - ``~/.ssh/id_dsa[.pub]`` is used as the cluster key. It is assumed that - this key is authorized to ssh to root@`hostname`. -- No service spec is defined for mon or mgr, which means that cephadm - does not try to manage them. + this key is authorized to ssh with no passphrase to root@`hostname`. +- cephadm does not try to manage any daemons started by vstart.sh (any + nonzero number in the environment variables). No service spec is defined + for mon or mgr. - You'll see health warnings from cephadm about stray daemons--that's because - the vstart-launched mon and mgr aren't controlled by cephadm. + the vstart-launched daemons aren't controlled by cephadm. - The default image is ``quay.io/ceph-ci/ceph:master``, but you can change this by passing ``-o container_image=...`` or ``ceph config set global container_image ...``. @@ -89,14 +90,15 @@ When you're done, you can tear down the cluster with:: Note regarding network calls from CLI handlers ============================================== -Executing any cephadm CLI commands like ``ceph orch ls`` will block -the mon command handler thread within the MGR, thus preventing any -concurrent CLI calls. Note that pressing ``^C`` will not resolve this -situation, as *only* the client will be aborted, but not exceution -itself. This means, cephadm will be completely unresonsive, until the -execution of the CLI handler is fully completed. Note that even -``ceph orch ps`` will not respond, while another handler is executed. - -This means, we should only do very few calls to remote hosts synchronously. -As a guideline, cephadm should do at most ``O(1)`` network calls in CLI handlers. +Executing any cephadm CLI commands like ``ceph orch ls`` will block the +mon command handler thread within the MGR, thus preventing any concurrent +CLI calls. Note that pressing ``^C`` will not resolve this situation, +as *only* the client will be aborted, but not execution of the command +within the orchestrator manager module itself. This means, cephadm will +be completely unresponsive until the execution of the CLI handler is +fully completed. Note that even ``ceph orch ps`` will not respond while +another handler is executing. + +This means we should do very few synchronous calls to remote hosts. +As a guideline, cephadm should do at most ``O(1)`` network calls in CLI handlers. Everything else should be done asynchronously in other threads, like ``serve()``. diff --git a/ceph/doc/dev/developer_guide/running-tests-locally.rst b/ceph/doc/dev/developer_guide/running-tests-locally.rst index f80a6f52b..49eff3ece 100644 --- a/ceph/doc/dev/developer_guide/running-tests-locally.rst +++ b/ceph/doc/dev/developer_guide/running-tests-locally.rst @@ -36,6 +36,8 @@ This means the cluster is running. Step 3 - run s3-tests ^^^^^^^^^^^^^^^^^^^^^ +.. highlight:: console + To run the s3tests suite do the following:: $ ../qa/workunits/rgw/run-s3tests.sh @@ -50,17 +52,12 @@ Running your first test The Python tests in Ceph repository can be executed on your local machine using `vstart_runner.py`_. To do that, you'd need `teuthology`_ installed:: - $ git clone https://github.com/ceph/teuthology - $ cd teuthology/ - $ virtualenv -p python2.7 ./venv + $ virtualenv --python=python3 venv $ source venv/bin/activate - $ pip install --upgrade pip - $ pip install -r requirements.txt - $ python setup.py develop + $ pip install 'setuptools >= 12' + $ pip install git+https://github.com/ceph/teuthology#egg=teuthology[test] $ deactivate -.. note:: The pip command above is pip2, not pip3; run ``pip --version``. - The above steps installs teuthology in a virtual environment. Before running a test locally, build Ceph successfully from the source (refer :doc:`/install/build-ceph`) and do:: @@ -73,24 +70,20 @@ To run a specific test, say `test_reconnect_timeout`_ from `TestClientRecovery`_ in ``qa/tasks/cephfs/test_client_recovery``, you can do:: - $ python2 ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery.test_reconnect_timeout + $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery.test_reconnect_timeout The above command runs vstart_runner.py and passes the test to be executed as an argument to vstart_runner.py. In a similar way, you can also run the group of tests in the following manner:: $ # run all tests in class TestClientRecovery - $ python2 ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery + $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery $ # run all tests in test_client_recovery.py - $ python2 ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery + $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery Based on the argument passed, vstart_runner.py collects tests and executes as it would execute a single test. -.. note:: vstart_runner.py as well as most tests in ``qa/`` are only - compatible with ``python2``. Therefore, use ``python2`` to run the - tests locally. - vstart_runner.py can take the following options - --clear-old-log deletes old log file before running the test diff --git a/ceph/doc/dev/msgr2.rst b/ceph/doc/dev/msgr2.rst index a5a484018..0eed9e67f 100644 --- a/ceph/doc/dev/msgr2.rst +++ b/ceph/doc/dev/msgr2.rst @@ -1,7 +1,7 @@ .. _msgr2-protocol: -msgr2 protocol -============== +msgr2 protocol (msgr2.0 and msgr2.1) +==================================== This is a revision of the legacy Ceph on-wire protocol that was implemented by the SimpleMessenger. It addresses performance and @@ -20,7 +20,7 @@ This protocol revision has several goals relative to the original protocol: (e.g., padding) that keep computation and memory copies out of the fast path where possible. * *Signing*. We will allow for traffic to be signed (but not - necessarily encrypted). This may not be implemented in the initial version. + necessarily encrypted). This is not implemented. Definitions ----------- @@ -56,10 +56,19 @@ Banner Both the client and server, upon connecting, send a banner:: - "ceph %x %x\n", protocol_features_suppored, protocol_features_required + "ceph v2\n" + __le16 banner payload length + banner payload -The protocol features are a new, distinct namespace. Initially no -features are defined or required, so this will be "ceph 0 0\n". +A banner payload has the form:: + + __le64 peer_supported_features + __le64 peer_required_features + +This is a new, distinct feature bit namespace (CEPH_MSGR2_*). +Currently, only CEPH_MSGR2_FEATURE_REVISION_1 is defined. It is +supported but not required, so that msgr2.0 and msgr2.1 peers +can talk to each other. If the remote party advertises required features we don't support, we can disconnect. @@ -79,27 +88,150 @@ can disconnect. Frame format ------------ -All further data sent or received is contained by a frame. Each frame has -the form:: +After the banners are exchanged, all further communication happens +in frames. The exact format of the frame depends on the connection +mode (msgr2.0-crc, msgr2.0-secure, msgr2.1-crc or msgr2.1-secure). +All connections start in crc mode (either msgr2.0-crc or msgr2.1-crc, +depending on peer_supported_features from the banner). + +Each frame has a 32-byte preamble:: + + __u8 tag + __u8 number of segments + { + __le32 segment length + __le16 segment alignment + } * 4 + reserved (2 bytes) + __le32 preamble crc + +An empty frame has one empty segment. A non-empty frame can have +between one and four segments, all segments except the last may be +empty. + +If there are less than four segments, unused (trailing) segment +length and segment alignment fields are zeroed. + +The reserved bytes are zeroed. + +The preamble checksum is CRC32-C. It covers everything up to +itself (28 bytes) and is calculated and verified irrespective of +the connection mode (i.e. even if the frame is encrypted). + +### msgr2.0-crc mode + +A msgr2.0-crc frame has the form:: + + preamble (32 bytes) + { + segment payload + } * number of segments + epilogue (17 bytes) + +where epilogue is:: + + __u8 late_flags + { + __le32 segment crc + } * 4 + +late_flags is used for frame abortion. After transmitting the +preamble and the first segment, the sender can fill the remaining +segments with zeros and set a flag to indicate that the receiver must +drop the frame. This allows the sender to avoid extra buffering +when a frame that is being put on the wire is revoked (i.e. yanked +out of the messenger): payload buffers can be unpinned and handed +back to the user immediately, without making a copy or blocking +until the whole frame is transmitted. Currently this is used only +by the kernel client, see ceph_msg_revoke(). + +The segment checksum is CRC32-C. For "used" empty segments, it is +set to (__le32)-1. For unused (trailing) segments, it is zeroed. + +The crcs are calculated just to protect against bit errors. +No authenticity guarantees are provided, unlike in msgr1 which +attempted to provide some authenticity guarantee by optionally +signing segment lengths and crcs with the session key. + +Issues: + +1. As part of introducing a structure for a generic frame with + variable number of segments suitable for both control and + message frames, msgr2.0 moved the crc of the first segment of + the message frame (ceph_msg_header2) into the epilogue. + + As a result, ceph_msg_header2 can no longer be safely + interpreted before the whole frame is read off the wire. + This is a regression from msgr1, because in order to scatter + the payload directly into user-provided buffers and thus avoid + extra buffering and copying when receiving message frames, + ceph_msg_header2 must be available in advance -- it stores + the transaction id which the user buffers are keyed on. + The implementation has to choose between forgoing this + optimization or acting on an unverified segment. + +2. late_flags is not covered by any crc. Since it stores the + abort flag, a single bit flip can result in a completed frame + being dropped (causing the sender to hang waiting for a reply) + or, worse, in an aborted frame with garbage segment payloads + being dispatched. + + This was the case with msgr1 and got carried over to msgr2.0. + +### msgr2.1-crc mode + +Differences from msgr2.0-crc: + +1. The crc of the first segment is stored at the end of the + first segment, not in the epilogue. The epilogue stores up to + three crcs, not up to four. + + If the first segment is empty, (__le32)-1 crc is not generated. + +2. The epilogue is generated only if the frame has more than one + segment (i.e. at least one of second to fourth segments is not + empty). Rationale: If the frame has only one segment, it cannot + be aborted and there are no crcs to store in the epilogue. - frame_len (le32) - tag (TAG_* le32) - frame_header_checksum (le32) - payload - [payload padding -- only present after stream auth phase] - [signature -- only present after stream auth phase] +3. Unchecksummed late_flags is replaced with late_status which + builds in bit error detection by using a 4-bit nibble per flag + and two code words that are Hamming Distance = 4 apart (and not + all zeros or ones). This comes at the expense of having only + one reserved flag, of course. +Some example frames: -* The frame_header_checksum is over just the frame_len and tag values (8 bytes). +* A 0+0+0+0 frame (empty, no epilogue):: -* frame_len includes everything after the frame_len le32 up to the end of the - frame (all payloads, signatures, and padding). + preamble (32 bytes) -* The payload format and length is determined by the tag. +* A 20+0+0+0 frame (no epilogue):: -* The signature portion is only present if the authentication phase - has completed (TAG_AUTH_DONE has been sent) and signatures are - enabled. + preamble (32 bytes) + segment1 payload (20 bytes) + __le32 segment1 crc + +* A 0+70+0+0 frame:: + + preamble (32 bytes) + segment2 payload (70 bytes) + epilogue (13 bytes) + +* A 20+70+0+350 frame:: + + preamble (32 bytes) + segment1 payload (20 bytes) + __le32 segment1 crc + segment2 payload (70 bytes) + segment4 payload (350 bytes) + epilogue (13 bytes) + +where epilogue is:: + + __u8 late_status + { + __le32 segment crc + } * 3 Hello ----- @@ -198,47 +330,197 @@ authentication method as the first attempt: Post-auth frame format ---------------------- -The frame format is fixed (see above), but can take three different -forms, depending on the AUTH_DONE flags: +Depending on the negotiated connection mode from TAG_AUTH_DONE, the +connection either stays in crc mode or switches to the corresponding +secure mode (msgr2.0-secure or msgr2.1-secure). + +### msgr2.0-secure mode + +A msgr2.0-secure frame has the form:: + + { + preamble (32 bytes) + { + segment payload + zero padding (out to 16 bytes) + } * number of segments + epilogue (16 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + +where epilogue is:: + + __u8 late_flags + zero padding (15 bytes) + +late_flags has the same meaning as in msgr2.0-crc mode. + +Each segment and the epilogue are zero padded out to 16 bytes. +Technically, GCM doesn't require any padding because Counter mode +(the C in GCM) essentially turns a block cipher into a stream cipher. +But, if the overall input length is not a multiple of 16 bytes, some +implicit zero padding would occur internally because GHASH function +used by GCM for generating auth tags only works on 16-byte blocks. + +Issues: + +1. The sender encrypts the whole frame using a single nonce + and generating a single auth tag. Because segment lengths are + stored in the preamble, the receiver has no choice but to decrypt + and interpret the preamble without verifying the auth tag -- it + can't even tell how much to read off the wire to get the auth tag + otherwise! This creates a decryption oracle, which, in conjunction + with Counter mode malleability, could lead to recovery of sensitive + information. + + This issue extends to the first segment of the message frame as + well. As in msgr2.0-crc mode, ceph_msg_header2 cannot be safely + interpreted before the whole frame is read off the wire. + +2. Deterministic nonce construction with a 4-byte counter field + followed by an 8-byte fixed field is used. The initial values are + taken from the connection secret -- a random byte string generated + during the authentication phase. Because the counter field is + only four bytes long, it can wrap and then repeat in under a day, + leading to GCM nonce reuse and therefore a potential complete + loss of both authenticity and confidentiality for the connection. + This was addressed by disconnecting before the counter repeats + (CVE-2020-1759). + +### msgr2.1-secure mode + +Differences from msgr2.0-secure: + +1. The preamble, the first segment and the rest of the frame are + encrypted separately, using separate nonces and generating + separate auth tags. This gets rid of unverified plaintext use + and keeps msgr2.1-secure mode close to msgr2.1-crc mode, allowing + the implementation to receive message frames in a similar fashion + (little to no buffering, same scatter/gather logic, etc). + + In order to reduce the number of en/decryption operations per + frame, the preamble is grown by a fixed size inline buffer (48 + bytes) that the first segment is inlined into, either fully or + partially. The preamble auth tag covers both the preamble and the + inline buffer, so if the first segment is small enough to be fully + inlined, it becomes available after a single decryption operation. + +2. As in msgr2.1-crc mode, the epilogue is generated only if the + frame has more than one segment. The rationale is even stronger, + as it would require an extra en/decryption operation. + +3. For consistency with msgr2.1-crc mode, late_flags is replaced + with late_status (the built-in bit error detection isn't really + needed in secure mode). + +4. In accordance with `NIST Recommendation for GCM`_, deterministic + nonce construction with a 4-byte fixed field followed by an 8-byte + counter field is used. An 8-byte counter field should never repeat + but the nonce reuse protection put in place for msgr2.0-secure mode + is still there. + + The initial values are the same as in msgr2.0-secure mode. + + .. _`NIST Recommendation for GCM`: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf + +As in msgr2.0-secure mode, each segment is zero padded out to +16 bytes. If the first segment is fully inlined, its padding goes +to the inline buffer. Otherwise, the padding is on the remainder. +The corollary to this is that the inline buffer is consumed in +16-byte chunks. + +The unused portion of the inline buffer is zeroed. + +Some example frames: + +* A 0+0+0+0 frame (empty, nothing to inline, no epilogue):: + + { + preamble (32 bytes) + zero padding (48 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + +* A 20+0+0+0 frame (first segment fully inlined, no epilogue):: + + { + preamble (32 bytes) + segment1 payload (20 bytes) + zero padding (28 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) -* If neither FLAG_SIGNED or FLAG_ENCRYPTED is specified, things are simple:: +* A 0+70+0+0 frame (nothing to inline):: - frame_len - tag - payload - payload_padding (out to auth block_size) + { + preamble (32 bytes) + zero padding (48 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + { + segment2 payload (70 bytes) + zero padding (10 bytes) + epilogue (16 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) - - The padding is some number of bytes < the auth block_size that - brings the total length of the payload + payload_padding to a - multiple of block_size. It does not include the frame_len or tag. Padding - content can be zeros or (better) random bytes. +* A 20+70+0+350 frame (first segment fully inlined):: -* If FLAG_SIGNED has been specified:: + { + preamble (32 bytes) + segment1 payload (20 bytes) + zero padding (28 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + { + segment2 payload (70 bytes) + zero padding (10 bytes) + segment4 payload (350 bytes) + zero padding (2 bytes) + epilogue (16 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) - frame_len - tag - payload - payload_padding (out to auth block_size) - signature (sig_size bytes) +* A 105+0+0+0 frame (first segment partially inlined, no epilogue):: - Here the padding just makes life easier for the signature. It can be - random data to add additional confounder. Note also that the - signature input must include some state from the session key and the - previous message. + { + preamble (32 bytes) + segment1 payload (48 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + { + segment1 payload remainder (57 bytes) + zero padding (7 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) -* If FLAG_ENCRYPTED has been specified:: +* A 105+70+0+350 frame (first segment partially inlined):: - frame_len - tag { - payload - payload_padding (out to auth block_size) - } ^ stream cipher + preamble (32 bytes) + segment1 payload (48 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + { + segment1 payload remainder (57 bytes) + zero padding (7 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + { + segment2 payload (70 bytes) + zero padding (10 bytes) + segment4 payload (350 bytes) + zero padding (2 bytes) + epilogue (16 bytes) + } ^ AES-128-GCM cipher + auth tag (16 bytes) + +where epilogue is:: - Note that the padding ensures that the total frame is a multiple of - the auth method's block_size so that the message can be sent out over - the wire without waiting for the next frame in the stream. + __u8 late_status + zero padding (15 bytes) +late_status has the same meaning as in msgr2.1-crc mode. Message flow handshake ---------------------- diff --git a/ceph/doc/man/8/ceph-bluestore-tool.rst b/ceph/doc/man/8/ceph-bluestore-tool.rst index 2bc951ebd..af56cd759 100644 --- a/ceph/doc/man/8/ceph-bluestore-tool.rst +++ b/ceph/doc/man/8/ceph-bluestore-tool.rst @@ -158,6 +158,21 @@ BlueStore OSD with the *prime-osd-dir* command:: ceph-bluestore-tool prime-osd-dir --dev *main device* --path /var/lib/ceph/osd/ceph-*id* +BlueFS log rescue +===================== + +Some versions of BlueStore were susceptible to BlueFS log growing extremaly large - +beyond the point of making booting OSD impossible. This state is indicated by +booting that takes very long and fails in _replay function. + +This can be fixed by:: + ceph-bluestore-tool fsck --path *osd path* --bluefs_replay_recovery=true + +It is advised to first check if rescue process would be successfull:: + ceph-bluestore-tool fsck --path *osd path* \ + --bluefs_replay_recovery=true --bluefs_replay_recovery_disable_compact=true + +If above fsck is successfull fix procedure can be applied. Availability ============ diff --git a/ceph/doc/man/8/cephadm.rst b/ceph/doc/man/8/cephadm.rst index bd5fa8698..72bbf22ed 100644 --- a/ceph/doc/man/8/cephadm.rst +++ b/ceph/doc/man/8/cephadm.rst @@ -37,7 +37,7 @@ Synopsis | **cephadm** **run** [-h] --name NAME --fsid FSID | **cephadm** **shell** [-h] [--fsid FSID] [--name NAME] [--config CONFIG] - [--keyring KEYRING] [--env ENV] + [--keyring KEYRING] [--mount MOUNT] [--env ENV] [--] [command [command ...]] | **cephadm** **enter** [-h] [--fsid FSID] --name NAME [command [command ...]] @@ -60,24 +60,30 @@ Synopsis | [--skip-ssh] | [--initial-dashboard-user INITIAL_DASHBOARD_USER] | [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] +| [--ssl-dashboard-port SSL_DASHBOARD_PORT] | [--dashboard-key DASHBOARD_KEY] | [--dashboard-crt DASHBOARD_CRT] | [--ssh-config SSH_CONFIG] | [--ssh-private-key SSH_PRIVATE_KEY] -| [--ssh-public-key SSH_PUBLIC_KEY] [--skip-mon-network] +| [--ssh-public-key SSH_PUBLIC_KEY] +| [--ssh-user SSH_USER] [--skip-mon-network] | [--skip-dashboard] [--dashboard-password-noupdate] | [--no-minimize-config] [--skip-ping-check] | [--skip-pull] [--skip-firewalld] [--allow-overwrite] | [--allow-fqdn-hostname] [--skip-prepare-host] | [--orphan-initial-daemons] [--skip-monitoring-stack] | [--apply-spec APPLY_SPEC] +| [--registry-url REGISTRY_URL] +| [--registry-username REGISTRY_USERNAME] +| [--registry-password REGISTRY_PASSWORD] +| [--registry-json REGISTRY_JSON] | **cephadm** **deploy** [-h] --name NAME --fsid FSID [--config CONFIG] | [--config-json CONFIG_JSON] [--keyring KEYRING] | [--key KEY] [--osd-fsid OSD_FSID] [--skip-firewalld] -| [--reconfig] [--allow-ptrace] +| [--tcp-ports TCP_PORTS] [--reconfig] [--allow-ptrace] | **cephadm** **check-host** [-h] [--expect-hostname EXPECT_HOSTNAME] @@ -92,6 +98,10 @@ Synopsis | **cephadm** **install** [-h] [packages [packages ...]] +| **cephadm** **registry-login** [-h] [--registry-url REGISTRY_URL] +| [--registry-username REGISTRY_USERNAME] +| [--registry-password REGISTRY_PASSWORD] +| [--registry-json REGISTRY_JSON] [--fsid FSID] @@ -201,11 +211,13 @@ Arguments: * [--skip-ssh skip setup of ssh key on local host * [--initial-dashboard-user INITIAL_DASHBOARD_USER] Initial user for the dashboard * [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] Initial password for the initial dashboard user +* [--ssl-dashboard-port SSL_DASHBOARD_PORT] Port number used to connect with dashboard using SSL * [--dashboard-key DASHBOARD_KEY] Dashboard key * [--dashboard-crt DASHBOARD_CRT] Dashboard certificate * [--ssh-config SSH_CONFIG] SSH config * [--ssh-private-key SSH_PRIVATE_KEY] SSH private key * [--ssh-public-key SSH_PUBLIC_KEY] SSH public key +* [--ssh-user SSH_USER] set user for SSHing to cluster hosts, passwordless sudo will be needed for non-root users' * [--skip-mon-network] set mon public_network based on bootstrap mon ip * [--skip-dashboard] do not enable the Ceph Dashboard * [--dashboard-password-noupdate] stop forced dashboard password change @@ -219,6 +231,10 @@ Arguments: * [--orphan-initial-daemons] Do not create initial mon, mgr, and crash service specs * [--skip-monitoring-stack] Do not automatically provision monitoring stack] (prometheus, grafana, alertmanager, node-exporter) * [--apply-spec APPLY_SPEC] Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services) +* [--registry-url REGISTRY_URL] url of custom registry to login to. e.g. docker.io, quay.io +* [--registry-username REGISTRY_USERNAME] username of account to login to on custom registry +* [--registry-password REGISTRY_PASSWORD] password of account to login to on custom registry +* [--registry-json REGISTRY_JSON] JSON file containing registry login info (see registry-login command documentation) ceph-volume ----------- @@ -265,6 +281,7 @@ Arguments: * [--key KEY] key for new daemon * [--osd-fsid OSD_FSID] OSD uuid, if creating an OSD container * [--skip-firewalld] Do not configure firewalld +* [--tcp-ports List of tcp ports to open in the host firewall * [--reconfig] Reconfigure a previously deployed daemon * [--allow-ptrace] Allow SYS_PTRACE on daemon container @@ -358,6 +375,34 @@ Pull the ceph image:: cephadm pull +registry-login +-------------- + +Give cephadm login information for an authenticated registry (url, username and password). +Cephadm will attempt to log the calling host into that registry:: + + cephadm registry-login --registry-url [REGISTRY_URL] --registry-username [USERNAME] + --registry-password [PASSWORD] + +Can also use a JSON file containing the login info formatted as:: + + { + "url":"REGISTRY_URL", + "username":"REGISTRY_USERNAME", + "password":"REGISTRY_PASSWORD" + } + +and turn it in with command:: + + cephadm registry-login --registry-json [JSON FILE] + +Arguments: + +* [--registry-url REGISTRY_URL] url of registry to login to. e.g. docker.io, quay.io +* [--registry-username REGISTRY_USERNAME] username of account to login to on registry +* [--registry-password REGISTRY_PASSWORD] password of account to login to on registry +* [--registry-json REGISTRY_JSON] JSON file containing login info for custom registry +* [--fsid FSID] cluster FSID rm-daemon --------- @@ -420,6 +465,7 @@ Arguments: * [--name NAME, -n NAME] daemon name (type.id) * [--config CONFIG, -c CONFIG] ceph.conf to pass through to the container * [--keyring KEYRING, -k KEYRING] ceph.keyring to pass through to the container +* [--mount MOUNT, -m MOUNT] mount a file or directory under /mnt in the container * [--env ENV, -e ENV] set environment variable diff --git a/ceph/doc/man/8/rbd.rst b/ceph/doc/man/8/rbd.rst index b5a373584..52a684fcd 100644 --- a/ceph/doc/man/8/rbd.rst +++ b/ceph/doc/man/8/rbd.rst @@ -792,6 +792,57 @@ Per mapping (block device) `rbd device map` options: solid-state drives). For filestore with filestore_punch_hole = false, the recommended setting is image object size (typically 4M). +* crush_location=x - Specify the location of the client in terms of CRUSH + hierarchy (since 5.8). This is a set of key-value pairs separated from + each other by '|', with keys separated from values by ':'. Note that '|' + may need to be quoted or escaped to avoid it being interpreted as a pipe + by the shell. The key is the bucket type name (e.g. rack, datacenter or + region with default bucket types) and the value is the bucket name. For + example, to indicate that the client is local to rack "myrack", data center + "mydc" and region "myregion":: + + crush_location=rack:myrack|datacenter:mydc|region:myregion + + Each key-value pair stands on its own: "myrack" doesn't need to reside in + "mydc", which in turn doesn't need to reside in "myregion". The location + is not a path to the root of the hierarchy but rather a set of nodes that + are matched independently, owning to the fact that bucket names are unique + within a CRUSH map. "Multipath" locations are supported, so it is possible + to indicate locality for multiple parallel hierarchies:: + + crush_location=rack:myrack1|rack:myrack2|datacenter:mydc + +* read_from_replica=no - Disable replica reads, always pick the primary OSD + (since 5.8, default). + +* read_from_replica=balance - When issued a read on a replicated pool, pick + a random OSD for serving it (since 5.8). + + This mode is safe for general use only since Octopus (i.e. after "ceph osd + require-osd-release octopus"). Otherwise it should be limited to read-only + workloads such as images mapped read-only everywhere or snapshots. + +* read_from_replica=localize - When issued a read on a replicated pool, pick + the most local OSD for serving it (since 5.8). The locality metric is + calculated against the location of the client given with crush_location; + a match with the lowest-valued bucket type wins. For example, with default + bucket types, an OSD in a matching rack is closer than an OSD in a matching + data center, which in turn is closer than an OSD in a matching region. + + This mode is safe for general use only since Octopus (i.e. after "ceph osd + require-osd-release octopus"). Otherwise it should be limited to read-only + workloads such as images mapped read-only everywhere or snapshots. + +* compression_hint=none - Don't set compression hints (since 5.8, default). + +* compression_hint=compressible - Hint to the underlying OSD object store + backend that the data is compressible, enabling compression in passive mode + (since 5.8). + +* compression_hint=incompressible - Hint to the underlying OSD object store + backend that the data is incompressible, disabling compression in aggressive + mode (since 5.8). + `rbd device unmap` options: * force - Force the unmapping of a block device that is open (since 4.9). The diff --git a/ceph/doc/mgr/crash.rst b/ceph/doc/mgr/crash.rst index 76e0ce94a..e12a8c6f8 100644 --- a/ceph/doc/mgr/crash.rst +++ b/ceph/doc/mgr/crash.rst @@ -29,7 +29,7 @@ and will read from stdin. :: - ceph rm + ceph crash rm Remove a specific crash dump. diff --git a/ceph/doc/mgr/index.rst b/ceph/doc/mgr/index.rst index f6c85cec7..d59a13692 100644 --- a/ceph/doc/mgr/index.rst +++ b/ceph/doc/mgr/index.rst @@ -40,7 +40,6 @@ sensible. Telegraf module Telemetry module Iostat module - OSD Support module Crash module Insights module Orchestrator module diff --git a/ceph/doc/mgr/orchestrator.rst b/ceph/doc/mgr/orchestrator.rst index 6d3fc235f..5da26429f 100644 --- a/ceph/doc/mgr/orchestrator.rst +++ b/ceph/doc/mgr/orchestrator.rst @@ -59,8 +59,6 @@ Status Show current orchestrator mode and high-level status (whether the module able to talk to it) -Also show any in-progress actions. - Host Management =============== @@ -73,6 +71,8 @@ Add and remove hosts:: ceph orch host add [] [...] ceph orch host rm +For cephadm, see also :ref:`cephadm-fqdn`. + Host Specification ------------------ @@ -142,57 +142,69 @@ Example command:: ceph orch device zap my_hostname /dev/sdx +.. note:: + Cephadm orchestrator will automatically deploy drives that match the DriveGroup in your OSDSpec if the unmanaged flag is unset. + For example, if you use the ``all-available-devices`` option when creating OSD's, when you ``zap`` a device the cephadm orchestrator will automatically create a new OSD in the device . + To disable this behavior, see :ref:`orchestrator-cli-create-osds`. + +.. _orchestrator-cli-create-osds: Create OSDs ----------- -Create OSDs on a group of devices on a single host:: +Create OSDs on a set of devices on a single host:: ceph orch daemon add osd :device1,device2 -or:: - - ceph orch apply osd -i - - -or:: - - ceph orch apply osd --all-available-devices +Another way of doing it is using ``apply`` interface:: + ceph orch apply osd -i [--dry-run] +Where the ``json_file/yaml_file`` is a DriveGroup specification. For a more in-depth guide to DriveGroups please refer to :ref:`drivegroups` -Example:: - - # ceph orch daemon add osd node1:/dev/vdd - Created osd(s) 6 on host 'node1' - - -If the 'apply' method is used. You will be presented with a preview of what will happen. +Along with ``apply`` interface if ``dry-run`` option is used, it will present a +preview of what will happen. Example:: - # ceph orch apply osd --all-available-devices + # ceph orch apply osd --all-available-devices --dry-run NAME HOST DATA DB WAL all-available-devices node1 /dev/vdb - - all-available-devices node2 /dev/vdc - - all-available-devices node3 /dev/vdd - - - .. note:: - Output form Cephadm orchestrator + Example output from cephadm orchestrator + +When the parameter ``all-available-devices`` or a DriveGroup specification is used, a cephadm service is created. +This service guarantees that all available devices or devices included in the DriveGroup will be used for OSD's. +Take into account the implications of this behavior, which is automatic and enabled by default. + +For example: + +After using:: + + ceph orch apply osd --all-available-devices + +* If you add new disks to the cluster they will automatically be used to create new OSD's. +* A new OSD will be created automatically if you remove an OSD and clean the LVM physical volume. + +If you want to avoid this behavior (disable automatic creation of OSD in available devices), use the ``unmanaged`` parameter:: + + ceph orch apply osd --all-available-devices --unmanaged=true Remove an OSD -------------------- +------------- :: - ceph orch osd rm ... [--replace] [--force] + ceph orch osd rm [--replace] [--force] -Removes one or more OSDs from the cluster. +Evacuates PGs from an OSD and removes it from the cluster. Example:: - # ceph orch osd rm 4 + # ceph orch osd rm 0 Scheduled OSD(s) for removal @@ -201,20 +213,40 @@ OSDs that are not safe-to-destroy will be rejected. You can query the state of the operation with:: # ceph orch osd rm status - NAME HOST PGS STARTED_AT - osd.7 node1 55 2020-04-22 19:28:38.785761 - osd.5 node3 3 2020-04-22 19:28:34.201685 - osd.3 node2 0 2020-04-22 19:28:34.201695 + OSD_ID HOST STATE PG_COUNT REPLACE FORCE STARTED_AT + 2 cephadm-dev done, waiting for purge 0 True False 2020-07-17 13:01:43.147684 + 3 cephadm-dev draining 17 False True 2020-07-17 13:01:45.162158 + 4 cephadm-dev started 42 False True 2020-07-17 13:01:45.162158 When no PGs are left on the osd, it will be decommissioned and removed from the cluster. +.. note:: + After removing an OSD, if you wipe the LVM physical volume in the device used by the removed OSD, a new OSD will be created. + Read information about the ``unmanaged`` parameter in :ref:`orchestrator-cli-create-osds`. + +Stopping OSD Removal +-------------------- + +You can stop the operation with + +:: + + ceph orch osd rm stop + +Example:: + + # ceph orch osd rm stop 4 + Stopped OSD(s) removal + +This will reset the initial state of the OSD and remove it from the queue. + Replace an OSD ------------------- :: - orch osd rm ... --replace [--force] + orch osd rm --replace [--force] Example:: @@ -232,25 +264,18 @@ The previously set the 'destroyed' flag is used to determined osd ids that will If you use OSDSpecs for osd deployment, your newly added disks will be assigned with the osd ids of their replaced counterpart, granted the new disk still match the OSDSpecs. -For assistance in this process you can use the 'preview' feature: - -Example:: - - - ceph orch apply osd --service-name --preview - NAME HOST DATA DB WAL - node1 /dev/vdb - - +For assistance in this process you can use the '--dry-run' feature: Tip: The name of your OSDSpec can be retrieved from **ceph orch ls** Alternatively, you can use your OSDSpec file:: - ceph orch apply osd -i --preview + ceph orch apply osd -i --dry-run NAME HOST DATA DB WAL node1 /dev/vdb - - -If this matches your anticipated behavior, just omit the --preview flag to execute the deployment. +If this matches your anticipated behavior, just omit the --dry-run flag to execute the deployment. .. @@ -289,13 +314,13 @@ error if it doesn't know how to do this transition. Update the number of monitor hosts:: - ceph orch apply mon [host, host:network...] + ceph orch apply mon [host, host:network...] [--dry-run] Each host can optionally specify a network for the monitor to listen on. Update the number of manager hosts:: - ceph orch apply mgr [host...] + ceph orch apply mgr [host...] [--dry-run] .. .. note:: @@ -323,7 +348,7 @@ services of a particular type via optional --type parameter Discover the status of a particular service or daemons:: ceph orch ls --service_type type --service_name [--refresh] - + Export the service specs known to the orchestrator as yaml in format that is compatible to ``ceph orch apply -i``:: @@ -336,7 +361,7 @@ Daemon Status Print a list of all daemons known to the orchestrator:: ceph orch ps [--hostname host] [--daemon_type type] [--service_name name] [--daemon_id id] [--format f] [--refresh] - + Query the status of a particular service instance (mon, osd, mds, rgw). For OSDs the id is the numeric OSD ID, for MDS services it is the file system name:: @@ -344,18 +369,18 @@ the id is the numeric OSD ID, for MDS services it is the file system name:: .. _orchestrator-cli-cephfs: - + Depoying CephFS =============== In order to set up a :term:`CephFS`, execute:: ceph fs volume create - -Where ``name`` is the name of the CephFS, ``placement`` is a + +Where ``name`` is the name of the CephFS, ``placement`` is a :ref:`orchestrator-cli-placement-spec`. - -This command will create the required Ceph pools, create the new + +This command will create the required Ceph pools, create the new CephFS, and deploy mds servers. Stateless services (MDS/RGW/NFS/rbd-mirror/iSCSI) @@ -369,49 +394,45 @@ The ``name`` parameter is an identifier of the group of instances: * a CephFS file system for a group of MDS daemons, * a zone name for a group of RGWs -Sizing: the ``size`` parameter gives the number of daemons in the cluster -(e.g. the number of MDS daemons for a particular CephFS file system). - Creating/growing/shrinking/removing services:: - ceph orch {mds,rgw} update [host…] - ceph orch {mds,rgw} add - ceph orch nfs update [host…] - ceph orch nfs add [--namespace=] - ceph orch {mds,rgw,nfs} rm + ceph orch apply mds [--placement=] [--dry-run] + ceph orch apply rgw [--subcluster=] [--port=] [--ssl] [--placement=] [--dry-run] + ceph orch apply nfs [--namespace=] [--placement=] [--dry-run] + ceph orch rm [--force] -e.g., ``ceph orch mds update myfs 3 host1 host2 host3`` +Where ``placement`` is a :ref:`orchestrator-cli-placement-spec`. -Start/stop/reload:: +e.g., ``ceph orch apply mds myfs --placement="3 host1 host2 host3"`` - ceph orch service {stop,start,reload} +Service Commands:: + + ceph orch - ceph orch daemon {start,stop,reload} - .. _orchestrator-cli-service-spec: - + Service Specification ===================== -As *Service Specification* is a data structure often represented as YAML +As *Service Specification* is a data structure often represented as YAML to specify the deployment of services. For example: .. code-block:: yaml service_type: rgw service_id: realm.zone - placement: - hosts: + placement: + hosts: - host1 - host2 - host3 spec: ... unmanaged: false - + Where the properties of a service specification are the following: * ``service_type`` is the type of the service. Needs to be either a Ceph - service (``mon``, ``crash``, ``mds``, ``mgr``, ``osd`` or + service (``mon``, ``crash``, ``mds``, ``mgr``, ``osd`` or ``rbd-mirror``), a gateway (``nfs`` or ``rgw``), or part of the monitoring stack (``alertmanager``, ``grafana``, ``node-exporter`` or ``prometheus``). @@ -435,8 +456,8 @@ an optional namespace: service_type: nfs service_id: mynfs - placement: - hosts: + placement: + hosts: - host1 - host2 spec: @@ -462,6 +483,7 @@ Many service specifications can then be applied at once using host_pattern: "mgr*" --- service_type: osd + service_id: default_drive_group placement: host_pattern: "osd*" data_devices: @@ -469,12 +491,12 @@ Many service specifications can then be applied at once using EOF .. _orchestrator-cli-placement-spec: - + Placement Specification ======================= In order to allow the orchestrator to deploy a *service*, it needs to -know how many and where it should deploy *daemons*. The orchestrator +know how many and where it should deploy *daemons*. The orchestrator defines a placement specification that can either be passed as a command line argument. Explicit placements @@ -483,22 +505,22 @@ Explicit placements Daemons can be explictly placed on hosts by simply specifying them:: orch apply prometheus "host1 host2 host3" - + Or in yaml: .. code-block:: yaml - + service_type: prometheus placement: - hosts: + hosts: - host1 - host2 - host3 - + MONs and other services may require some enhanced network specifications:: orch daemon add mon myhost:[v2:1.2.3.4:3000,v1:1.2.3.4:6789]=name - + Where ``[v2:1.2.3.4:3000,v1:1.2.3.4:6789]`` is the network address of the monitor and ``=name`` specifies the name of the new monitor. @@ -545,18 +567,18 @@ Or in yaml: placement: host_pattern: "*" - + Setting a limit --------------- By specifying ``count``, only that number of daemons will be created:: orch apply prometheus 3 - + To deploy *daemons* on a subset of hosts, also specify the count:: orch apply prometheus "2 host1 host2 host3" - + If the count is bigger than the amount of hosts, cephadm still deploys two daemons:: orch apply prometheus "3 host1 host2" @@ -568,7 +590,7 @@ Or in yaml: service_type: prometheus placement: count: 3 - + Or with hosts: .. code-block:: yaml @@ -576,10 +598,30 @@ Or with hosts: service_type: prometheus placement: count: 2 - hosts: + hosts: - host1 - host2 - - host3 + - host3 + +Updating Service Specifications +=============================== + +The Ceph Orchestrator maintains a declarative state of each +service in a ``ServiceSpec``. For certain operations, like updating +the RGW HTTP port, we need to update the existing +specification. + +1. List the current ``ServiceSpec``:: + + ceph orch ls --service_name= --export > myservice.yaml + +2. Update the yaml file:: + + vi myservice.yaml + +3. Apply the new ``ServiceSpec``:: + + ceph orch apply -i myservice.yaml [--dry-run] Configuring the Orchestrator CLI ================================ diff --git a/ceph/doc/mgr/osd_support.rst b/ceph/doc/mgr/osd_support.rst deleted file mode 100644 index fb1037c2b..000000000 --- a/ceph/doc/mgr/osd_support.rst +++ /dev/null @@ -1,61 +0,0 @@ -OSD Support Module -================== -The OSD Support module holds osd specific functionality that -is needed by different components like the orchestrators. - -In current scope: - -* osd draining - -Enabling --------- -When an orchestrator is used this should be enabled as a dependency. -(*currently only valid for the cephadm orchestrator*) - -The *osd_support* module is manually enabled with:: - - ceph mgr module enable osd_support - -Commands --------- - -Draining -######## - -This mode is for draining OSDs gracefully. `Draining` in this context means gracefully emptying out OSDs by setting their -weight to zero. An OSD is considered to be drained when no PGs are left. - -:: - - ceph osd drain $osd_id - -Takes a $osd_id and schedules it for draining. Since that process can take -quite some time, the operation will be executed in the background. To query the status -of the operation you can use: - -:: - - ceph osd drain status - -This gives you the status of all running operations in this format:: - - [{'osd_id': 0, 'pgs': 1234}, ..] - -If you wish to stop an OSD from being drained:: - - ceph osd drain stop [$osd_id] - -Stops all **scheduled** osd drain operations (not the operations that have been started already) -if no $osd_ids are given. If $osd_ids are present it only operates on them. -To stop and reset the weight of already started operations we need to save the initial weight -(see 'Ideas for improvement') - - -Ideas for improvement ----------------------- -- add health checks set_health_checks -- use objects to represent OSDs - - allows timestamps, trending information etc -- save osd drain state (at least the osd_ids in the mon store) - - resume after a mgr crash - - save the initial weight of a osd i.e. (set to initial weight on abort) diff --git a/ceph/doc/mgr/prometheus.rst b/ceph/doc/mgr/prometheus.rst index 8dbc44f52..87296be39 100644 --- a/ceph/doc/mgr/prometheus.rst +++ b/ceph/doc/mgr/prometheus.rst @@ -25,11 +25,65 @@ The *prometheus* module is enabled with:: Configuration ------------- -By default the module will accept HTTP requests on port ``9283`` on all -IPv4 and IPv6 addresses on the host. The port and listen address are both +.. note:: + + The Prometheus manager module needs to be restarted for configuration changes to be applied. + +By default the module will accept HTTP requests on port ``9283`` on all IPv4 +and IPv6 addresses on the host. The port and listen address are both configurable with ``ceph config-key set``, with keys -``mgr/prometheus/server_addr`` and ``mgr/prometheus/server_port``. -This port is registered with Prometheus's `registry `_. +``mgr/prometheus/server_addr`` and ``mgr/prometheus/server_port``. This port +is registered with Prometheus's `registry +`_. + +:: + + ceph config set mgr mgr/prometheus/server_addr 0.0.0.0 + ceph config set mgr mgr/prometheus/server_port 9283 + +.. warning:: + + The ``scrape_interval`` of this module should always be set to match + Prometheus' scrape interval to work properly and not cause any issues. + +The Prometheus manager module is, by default, configured with a scrape interval +of 15 seconds. The scrape interval in the module is used for caching purposes +and to determine when a cache is stale. + +It is not recommended to use a scrape interval below 10 seconds. It is +recommended to use 15 seconds as scrape interval, though, in some cases it +might be useful to increase the scrape interval. + +To set a different scrape interval in the Prometheus module, set +``scrape_interval`` to the desired value:: + + ceph config set mgr mgr/prometheus/scrape_interval 20 + +On large clusters (>1000 OSDs), the time to fetch the metrics may become +significant. Without the cache, the Prometheus manager module could, +especially in conjunction with multiple Prometheus instances, overload the +manager and lead to unresponsive or crashing Ceph manager instances. Hence, +the cache is enabled by default and cannot be disabled. This means that there +is a possibility that the cache becomes stale. The cache is considered stale +when the time to fetch the metrics from Ceph exceeds the configured +``scrape_interval``. + +If that is the case, **a warning will be logged** and the module will either + +* respond with a 503 HTTP status code (service unavailable) or, +* it will return the content of the cache, even though it might be stale. + +This behavior can be configured. By default, it will return a 503 HTTP status +code (service unavailable). You can set other options using the ``ceph config +set`` commands. + +To tell the module to respond with possibly stale data, set it to ``return``:: + + ceph config set mgr mgr/prometheus/stale_cache_strategy return + +To tell the module to respond with "service unavailable", set it to ``fail``:: + + ceph config set mgr mgr/prometheus/stale_cache_strategy fail .. _prometheus-rbd-io-statistics: @@ -62,7 +116,7 @@ Statistic names and labels ========================== The names of the stats are exactly as Ceph names them, with -illegal characters ``.``, ``-`` and ``::`` translated to ``_``, +illegal characters ``.``, ``-`` and ``::`` translated to ``_``, and ``ceph_`` prefixed to all names. @@ -75,7 +129,7 @@ rocksdb stats. The *cluster* statistics (i.e. those global to the Ceph cluster) -have labels appropriate to what they report on. For example, +have labels appropriate to what they report on. For example, metrics relating to pools have a ``pool_id`` label. @@ -109,7 +163,7 @@ Correlating drive statistics with node_exporter The prometheus output from Ceph is designed to be used in conjunction with the generic host monitoring from the Prometheus node_exporter. -To enable correlation of Ceph OSD statistics with node_exporter's +To enable correlation of Ceph OSD statistics with node_exporter's drive statistics, special series are output like this: :: diff --git a/ceph/doc/rados/configuration/osd-config-ref.rst b/ceph/doc/rados/configuration/osd-config-ref.rst index 8a33ca452..344599fe7 100644 --- a/ceph/doc/rados/configuration/osd-config-ref.rst +++ b/ceph/doc/rados/configuration/osd-config-ref.rst @@ -258,7 +258,7 @@ scrubbing operations. Already running scrubs will be continued. This might be useful to reduce load on busy clusters. :Type: Boolean -:Default: ``true`` +:Default: ``false`` ``osd scrub thread timeout`` diff --git a/ceph/doc/rados/operations/health-checks.rst b/ceph/doc/rados/operations/health-checks.rst index bd71a2ee9..4b3d5a7a2 100644 --- a/ceph/doc/rados/operations/health-checks.rst +++ b/ceph/doc/rados/operations/health-checks.rst @@ -688,6 +688,16 @@ paired with *PG_DAMAGED* (see above). See :doc:`pg-repair` for more information. +OSD_TOO_MANY_REPAIRS +____________________ + +When a read error occurs and another replica is available it is used to repair +the error immediately, so that the client can get the object data. Scrub +handles errors for data at rest. In order to identify possible failing disks +that aren't seeing scrub errors, a count of read repairs is maintained. If +it exceeds a config value threshold *mon_osd_warn_num_repaired* default 10, +this health warning is generated. + LARGE_OMAP_OBJECTS __________________ diff --git a/ceph/doc/radosgw/index.rst b/ceph/doc/radosgw/index.rst index 819986926..158ae41a9 100644 --- a/ceph/doc/radosgw/index.rst +++ b/ceph/doc/radosgw/index.rst @@ -47,6 +47,7 @@ you may write data with one API and retrieve it with the other. Config Reference Admin Guide S3 API + Data caching and CDN Swift API Admin Ops API Python binding diff --git a/ceph/doc/radosgw/opa.rst b/ceph/doc/radosgw/opa.rst index ef26a74ad..f1b76b5ef 100644 --- a/ceph/doc/radosgw/opa.rst +++ b/ceph/doc/radosgw/opa.rst @@ -46,6 +46,7 @@ Example request:: { "input": { "method": "GET", + "subuser": "subuser", "user_info": { "user_id": "john", "display_name": "John" diff --git a/ceph/doc/radosgw/rgw-cache.rst b/ceph/doc/radosgw/rgw-cache.rst new file mode 100644 index 000000000..d28a73887 --- /dev/null +++ b/ceph/doc/radosgw/rgw-cache.rst @@ -0,0 +1,127 @@ +========================== +RGW Data caching and CDN +========================== + +.. versionadded:: Octopus + +.. contents:: + +This feature adds to RGW the ability to securely cache objects and offload the workload from the cluster, using Nginx. +After an object is accessed the first time it will be stored in the Nginx directory. +When data is already cached, it need not be fetched from RGW. A permission check will be made against RGW to ensure the requesting user has access. +This feature is based on some Nginx modules, ngx_http_auth_request_module, https://github.com/kaltura/nginx-aws-auth-module, Openresty for lua capabilities. +Currently this feature only works for GET requests and it will cache only AWSv4 requests (only s3 requests). +The feature introduces 2 new APIs: Auth and Cache. + +New APIs +------------------------- + +There are 2 new apis for this feature: + +Auth API - The cache uses this to validate that an user can access the cached data + +Cache API - Adds the ability to override securely Range header, that way Nginx can use it is own smart cache on top of S3: +https://www.nginx.com/blog/smart-efficient-byte-range-caching-nginx/ +Using this API gives the ability to read ahead objects when clients asking a specific range from the object. +On subsequent accesses to the cached object, Nginx will satisfy requests for already-cached ranges from cache. Uncached ranges will be read from RGW (and cached). + +Auth API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API Validates a specific authenticated access being made to the cache, using RGW's knowledge of the client credentials and stored access policy. +Returns success if the encapsulated request would be granted. + +Cache API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API is meant to allow changing signed Range headers using a privileged user, cache user. + +Creating cache user + +:: + +$ radosgw-admin user create --uid= --display-name="cache user" --caps="amz-cache=read" + +This user can send to the RGW the Cache API header ``X-Amz-Cache``, this header contains the headers from the original request(before changing the Range header). +It means that ``X-Amz-Cache`` built from several headers. +The headers that are building the ``X-Amz-Cache`` header are separated by char with ascii code 177 and the header name and value are separated by char ascii code 178. +The RGW will check that the cache user is an authorized user and if it is a cache user, +if yes it will use the ``X-Amz-Cache`` to revalidate that the user have permissions, using the headers from the X-Amz-Cache. +During this flow the RGW will override the Range header. + + +Using Nginx with RGW +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Download the source of Openresty: + +:: + +$ wget https://openresty.org/download/openresty-1.15.8.3.tar.gz + +git clone the aws auth nginx module: + +:: + +$ git clone https://github.com/kaltura/nginx-aws-auth-module + +untar the openresty package: + +:: + +$ tar xvzf openresty-1.15.8.3.tar.gz +$ cd openresty-1.15.8.3 + +Compile openresty, Make sure that you have pcre lib and openssl lib: + +:: + +$ sudo yum install pcre-devel openssl-devel gcc curl zlib-devel nginx +$ ./configure --add-module= --with-http_auth_request_module --with-http_slice_module --conf-path=/etc/nginx/nginx.conf +$ gmake -j $(nproc) +$ sudo gmake install +$ sudo ln -sf /usr/local/openresty/bin/openresty /usr/bin/nginx + +Put in-place your nginx configuration files and edit them according to your environment: + +All nginx conf files are under: https://github.com/ceph/ceph/tree/master/examples/rgw-cache + +nginx.conf should go to /etc/nginx/nginx.conf + +nginx-lua-file.lua should go to /etc/nginx/nginx-lua-file.lua + +nginx-default.conf should go to /etc/nginx/conf.d/nginx-default.conf + +The parameters that are most likely to require adjustment according to the environment are located in the file nginx-default.conf + +Modify the example values of *proxy_cache_path* and *max_size* at: + +`proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;` + +And modify the example *server* values to point to the RGWs URIs: + +`server rgw1:8000 max_fails=2 fail_timeout=5s;` + +`server rgw2:8000 max_fails=2 fail_timeout=5s;` + +`server rgw3:8000 max_fails=2 fail_timeout=5s;` + +It is important to substitute the access key and secret key located in the nginx.conf with those belong to the user with the amz-cache caps + +It is possible to use nginx slicing which is a better method for streaming purposes. + +For using slice you should use nginx-slicing.conf and not nginx-default.conf + +Further information about nginx slicing: + +https://docs.nginx.com/nginx/admin-guide/content-cache/content-caching/#byte-range-caching + + +If you do not want to use the prefetch caching, It is possible to replace nginx-default.conf with nginx-noprefetch.conf +Using noprefetch means that if the client is sending range request of 0-4095 and then 0-4096 Nginx will cache those requests separately, So it will need to fetch those requests twice. + + +Run nginx(openresty): +:: + +$ sudo systemctl restart nginx diff --git a/ceph/doc/rbd/iscsi-target-cli.rst b/ceph/doc/rbd/iscsi-target-cli.rst index 2448a5414..7e9c56b3e 100644 --- a/ceph/doc/rbd/iscsi-target-cli.rst +++ b/ceph/doc/rbd/iscsi-target-cli.rst @@ -149,9 +149,14 @@ For rpm based instructions execute the following commands: :: # systemctl daemon-reload + + # systemctl enable rbd-target-gw + # systemctl start rbd-target-gw + # systemctl enable rbd-target-api # systemctl start rbd-target-api + **Configuring:** gwcli will create and configure the iSCSI target and RBD images and copy the diff --git a/ceph/doc/rbd/rbd-persistent-cache.rst b/ceph/doc/rbd/rbd-persistent-cache.rst index f7f104326..582ec8dae 100644 --- a/ceph/doc/rbd/rbd-persistent-cache.rst +++ b/ceph/doc/rbd/rbd-persistent-cache.rst @@ -71,6 +71,10 @@ not enough capacity it will delete some cold cache files. Here are some important cache options correspond to the following settings: +- ``immutable_object_cache_sock`` The path to the domain socket used for + communication between librbd clients and the ceph-immutable-object-cache + daemon. + - ``immutable_object_cache_path`` The immutable object cache data directory. - ``immutable_object_cache_max_size`` The max size for immutable cache. diff --git a/ceph/examples/rgw-cache/nginx-default.conf b/ceph/examples/rgw-cache/nginx-default.conf new file mode 100644 index 000000000..37dbb8070 --- /dev/null +++ b/ceph/examples/rgw-cache/nginx-default.conf @@ -0,0 +1,91 @@ +#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory +#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger +proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off; +upstream rgws { + # List of all rgws (ips or resolvable names) + server rgw1:8000 max_fails=2 fail_timeout=5s; + server rgw2:8000 max_fails=2 fail_timeout=5s; + server rgw3:8000 max_fails=2 fail_timeout=5s; +} +server { + listen 80; + server_name cacher; + location /authentication { + internal; + limit_except GET { deny all; } + proxy_pass http://rgws$request_uri; + proxy_pass_request_body off; + proxy_set_header Host $host; + # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data + proxy_set_header x-rgw-auth "yes"; + proxy_set_header Authorization $http_authorization; + proxy_http_version 1.1; + proxy_method $request_method; + # Do not convert HEAD requests into GET requests + proxy_cache_convert_head off; + error_page 404 = @outage; + proxy_intercept_errors on; + if ($request_uri = "/"){ + return 200; + } + # URI included with question mark is not being cached + if ($request_uri ~* (\?)){ + return 200; + } + } + location @outage{ + return 403; + } + location / { + limit_except GET { deny all; } + auth_request /authentication; + proxy_pass http://rgws; + set $authvar ''; + # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example + set $do_not_cache ''; + # the IP or name of the RGWs + rewrite_by_lua_file /etc/nginx/nginx-lua-file.lua; + #proxy_set_header Authorization $http_authorization; + # my cache configured at the top of the file + proxy_cache mycache; + proxy_cache_lock_timeout 0s; + proxy_cache_lock_age 1000s; + proxy_http_version 1.1; + set $date $aws_auth_date; + # Getting 403 if this header not set + proxy_set_header Host $host; + # Cache all 200 OK's for 1 day + proxy_cache_valid 200 206 1d; + # Use stale cache file in all errors from upstream if we can + proxy_cache_use_stale updating; + proxy_cache_background_update on; + # Try to check if etag have changed, if yes, do not re-fetch from rgw the object + proxy_cache_revalidate on; + # Lock the cache so that only one request can populate it at a time + proxy_cache_lock on; + # prevent convertion of head requests to get requests + proxy_cache_convert_head off; + # Listing all buckets should not be cached + if ($request_uri = "/"){ + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # URI including question mark are not supported to prevent bucket listing cache + if ($request_uri ~* (\?)){ + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # Only aws4 requests are being cached - As the aws auth module supporting only aws v2 + if ($http_authorization !~* "aws4_request") { + set $date $http_x_amz_date; + } + # Use the original x-amz-date if the aws auth module didn't create one + proxy_set_header x-amz-date $date; + proxy_set_header X-Amz-Cache $authvar; + proxy_no_cache $do_not_cache; + proxy_set_header Authorization $awsauth; + # This is on which content the nginx to use for hashing the cache keys + proxy_cache_key "$request_uri$request_method$request_body"; + client_max_body_size 20G; + } +} diff --git a/ceph/examples/rgw-cache/nginx-lua-file.lua b/ceph/examples/rgw-cache/nginx-lua-file.lua new file mode 100644 index 000000000..d776cb700 --- /dev/null +++ b/ceph/examples/rgw-cache/nginx-lua-file.lua @@ -0,0 +1,26 @@ +local check = ngx.req.get_headers()["AUTHORIZATION"] +local uri = ngx.var.request_uri +local ngx_re = require "ngx.re" +local hdrs = ngx.req.get_headers() +--Take all signedheaders names, this for creating the X-Amz-Cache which is necessary to override range header to be able to readahead an object +local res, err = ngx_re.split(check,"SignedHeaders=") +local res2, err2 = ngx_re.split(res[2],",") +local res3, err3 = ngx_re.split(res2[1],";") +local t = {} +local concathdrs = string.char(0x00) +for i = 1, #res3, 1 do + if hdrs[res3[i]] ~= nil then +--0xB1 is the separator between header name and value + t[i] = res3[i] .. string.char(0xB1) .. hdrs[res3[i]] +--0xB2 is the separator between headers + concathdrs = concathdrs .. string.char(0xB2) .. t[i] + end +end +-- check if the authorization header is not empty +if check ~= nil then + local xamzcache = concathdrs:sub(2) + xamzcache = xamzcache .. string.char(0xB2) .. "Authorization" .. string.char(0xB1) .. check + if xamzcache:find("aws4_request") ~= nil and uri ~= "/" and uri:find("?") == nil then + ngx.var.authvar = xamzcache + end +end diff --git a/ceph/examples/rgw-cache/nginx-noprefetch.conf b/ceph/examples/rgw-cache/nginx-noprefetch.conf new file mode 100644 index 000000000..30661d300 --- /dev/null +++ b/ceph/examples/rgw-cache/nginx-noprefetch.conf @@ -0,0 +1,81 @@ +#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory +#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger +proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off; +upstream rgws { + # List of all rgws (ips or resolvable names) + server rgw1:8000 max_fails=2 fail_timeout=5s; + server rgw2:8000 max_fails=2 fail_timeout=5s; + server rgw3:8000 max_fails=2 fail_timeout=5s; +} +server { + listen 80; + server_name cacher; + location /authentication { + internal; + limit_except GET { deny all; } + proxy_pass http://rgws$request_uri; + proxy_pass_request_body off; + proxy_set_header Host $host; + # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data + proxy_set_header x-rgw-auth "yes"; + proxy_set_header Authorization $http_authorization; + proxy_http_version 1.1; + proxy_method $request_method; + # Do not convert HEAD requests into GET requests + proxy_cache_convert_head off; + error_page 404 = @outage; + proxy_intercept_errors on; + if ($request_uri = "/"){ + return 200; + } + # URI included with question mark is not being cached + if ($request_uri ~* (\?)){ + return 200; + } + } + location @outage{ + return 403; + } + location / { + limit_except GET { deny all; } + auth_request /authentication; + proxy_pass http://rgws; + # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example + set $do_not_cache ''; + # the IP or name of the RGWs + #proxy_set_header Authorization $http_authorization; + # my cache configured at the top of the file + proxy_cache mycache; + proxy_cache_lock_timeout 0s; + proxy_cache_lock_age 1000s; + proxy_http_version 1.1; + # Getting 403 if this header not set + proxy_set_header Host $host; + # Cache all 200 OK's for 1 day + proxy_cache_valid 200 206 1d; + # Use stale cache file in all errors from upstream if we can + proxy_cache_use_stale updating; + proxy_cache_background_update on; + # Try to check if etag have changed, if yes, do not re-fetch from rgw the object + proxy_cache_revalidate on; + # Lock the cache so that only one request can populate it at a time + proxy_cache_lock on; + # prevent convertion of head requests to get requests + proxy_cache_convert_head off; + # Listing all buckets should not be cached + if ($request_uri = "/"){ + set $do_not_cache "no"; + } + # URI including question mark are not supported to prevent bucket listing cache + if ($request_uri ~* (\?)){ + set $do_not_cache "no"; + } + # Use the original x-amz-date if the aws auth module didn't create one + proxy_no_cache $do_not_cache; + proxy_set_header Authorization $http_authorization; + proxy_set_header Range $http_range; + # This is on which content the nginx to use for hashing the cache keys + proxy_cache_key "$request_uri$request_method$request_body$http_range"; + client_max_body_size 20G; + } +} diff --git a/ceph/examples/rgw-cache/nginx-slicing.conf b/ceph/examples/rgw-cache/nginx-slicing.conf new file mode 100644 index 000000000..1d6606d30 --- /dev/null +++ b/ceph/examples/rgw-cache/nginx-slicing.conf @@ -0,0 +1,93 @@ +#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory +#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger +proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off; +upstream rgws { + # List of all rgws (ips or resolvable names) + server rgw1:8000 max_fails=2 fail_timeout=5s; + server rgw2:8000 max_fails=2 fail_timeout=5s; + server rgw3:8000 max_fails=2 fail_timeout=5s; +} +server { + listen 80; + server_name cacher; + location /authentication { + internal; + limit_except GET { deny all; } + proxy_pass http://rgws$request_uri; + proxy_pass_request_body off; + proxy_set_header Host $host; + # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data + proxy_set_header x-rgw-auth "yes"; + proxy_set_header Authorization $http_authorization; + proxy_http_version 1.1; + proxy_method $request_method; + # Do not convert HEAD requests into GET requests + proxy_cache_convert_head off; + error_page 404 = @outage; + proxy_intercept_errors on; + if ($request_uri = "/"){ + return 200; + } + # URI included with question mark is not being cached + if ($request_uri ~* (\?)){ + return 200; + } + } + location @outage{ + return 403; + } + location / { + slice 1m; + limit_except GET { deny all; } + auth_request /authentication; + proxy_set_header Range $slice_range; + proxy_pass http://rgws; + set $authvar ''; + # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example + set $do_not_cache ''; + # the IP or name of the RGWs + rewrite_by_lua_file /etc/nginx/nginx-lua-file.lua; + #proxy_set_header Authorization $http_authorization; + # my cache configured at the top of the file + proxy_cache mycache; + proxy_cache_lock_timeout 0s; + proxy_cache_lock_age 1000s; + proxy_http_version 1.1; + set $date $aws_auth_date; + # Getting 403 if this header not set + proxy_set_header Host $host; + # Cache all 200 OK's for 1 day + proxy_cache_valid 200 206 1d; + # Use stale cache file in all errors from upstream if we can + proxy_cache_use_stale updating; + proxy_cache_background_update on; + # Try to check if etag have changed, if yes, do not re-fetch from rgw the object + proxy_cache_revalidate on; + # Lock the cache so that only one request can populate it at a time + proxy_cache_lock on; + # prevent convertion of head requests to get requests + proxy_cache_convert_head off; + # Listing all buckets should not be cached + if ($request_uri = "/"){ + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # URI including question mark are not supported to prevent bucket listing cache + if ($request_uri ~* (\?)){ + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # Only aws4 requests are being cached - As the aws auth module supporting only aws v2 + if ($http_authorization !~* "aws4_request") { + set $date $http_x_amz_date; + } + # Use the original x-amz-date if the aws auth module didn't create one + proxy_set_header x-amz-date $date; + proxy_set_header X-Amz-Cache $authvar; + proxy_no_cache $do_not_cache; + proxy_set_header Authorization $awsauth; + # This is on which content the nginx to use for hashing the cache keys + proxy_cache_key "$request_uri$request_method$request_body$slice_range"; + client_max_body_size 20G; + } +} diff --git a/ceph/examples/rgw-cache/nginx.conf b/ceph/examples/rgw-cache/nginx.conf new file mode 100644 index 000000000..f000597da --- /dev/null +++ b/ceph/examples/rgw-cache/nginx.conf @@ -0,0 +1,40 @@ + +user nginx; +#Process per core +worker_processes auto; +pid /var/run/nginx.pid; +events { +#Number of connections per worker + worker_connections 1024; +} + + +http { + types_hash_max_size 4096; + lua_package_path '/usr/local/openresty/lualib/?.lua;;'; + aws_auth $aws_token { + # access key and secret key of the cache + # Please substitute with the access key and secret key of the amz-cache cap user + access_key cache; + secret_key cache; + service s3; + region us-east-1; + } + # This map is used to choose the original authorization header if the aws_auth module refuse to create one + map $aws_token $awsauth { + default $http_authorization; + ~. $aws_token; # Regular expression to match any value + } + include /etc/nginx/mime.types; + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + error_log /var/log/nginx/error.log; + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nodelay on; + keepalive_timeout 65; + include /etc/nginx/conf.d/*.conf; +} diff --git a/ceph/monitoring/grafana/dashboards/hosts-overview.json b/ceph/monitoring/grafana/dashboards/hosts-overview.json index d9f1fb29d..804aa51cc 100644 --- a/ceph/monitoring/grafana/dashboards/hosts-overview.json +++ b/ceph/monitoring/grafana/dashboards/hosts-overview.json @@ -574,7 +574,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,( 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )\n)", + "expr": "topk(10,100 * ( 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", diff --git a/ceph/monitoring/grafana/dashboards/osds-overview.json b/ceph/monitoring/grafana/dashboards/osds-overview.json index 869fdc2fa..4b91df9eb 100644 --- a/ceph/monitoring/grafana/dashboards/osds-overview.json +++ b/ceph/monitoring/grafana/dashboards/osds-overview.json @@ -431,7 +431,7 @@ "strokeWidth": 1, "targets": [ { - "expr": "count by(device_class) (ceph_osd_metadata)", + "expr": "count by (device_class) (ceph_osd_metadata)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device_class}}", diff --git a/ceph/monitoring/grafana/dashboards/rbd-overview.json b/ceph/monitoring/grafana/dashboards/rbd-overview.json index f3df003ec..eb15fbcb8 100644 --- a/ceph/monitoring/grafana/dashboards/rbd-overview.json +++ b/ceph/monitoring/grafana/dashboards/rbd-overview.json @@ -416,7 +416,7 @@ ], "targets": [ { - "expr": "topk(10, (sort((irate(ceph_rbd_write_ops[30s]) + on(image, pool, namespace) irate(ceph_rbd_read_ops[30s])))))", + "expr": "topk(10, (sort((irate(ceph_rbd_write_ops[30s]) + on (image, pool, namespace) irate(ceph_rbd_read_ops[30s])))))", "format": "table", "instant": true, "intervalFactor": 1, diff --git a/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml b/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml index acafdd6d8..51d19bfca 100644 --- a/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml +++ b/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml @@ -232,7 +232,7 @@ groups: ( predict_linear(ceph_pool_stored[2d], 3600 * 24 * 5) >= ceph_pool_max_avail - ) * on(pool_id) group_right(name) ceph_pool_metadata + ) * on(pool_id) group_left(name) ceph_pool_metadata labels: severity: warning type: ceph_default diff --git a/ceph/qa/standalone/ceph-helpers.sh b/ceph/qa/standalone/ceph-helpers.sh index 19fede194..20336d06f 100755 --- a/ceph/qa/standalone/ceph-helpers.sh +++ b/ceph/qa/standalone/ceph-helpers.sh @@ -2069,6 +2069,10 @@ function flush_pg_stats() seqs='' for osd in $ids; do seq=`ceph tell osd.$osd flush_pg_stats` + if test -z "$seq" + then + continue + fi seqs="$seqs $osd-$seq" done diff --git a/ceph/qa/standalone/mon/mon-last-epoch-clean.sh b/ceph/qa/standalone/mon/mon-last-epoch-clean.sh index e38663c6a..667c6a702 100755 --- a/ceph/qa/standalone/mon/mon-last-epoch-clean.sh +++ b/ceph/qa/standalone/mon/mon-last-epoch-clean.sh @@ -181,8 +181,8 @@ function TEST_mon_last_clean_epoch() { sleep 5 - ceph tell osd.* injectargs '--osd-beacon-report-interval 10' || exit 1 - ceph tell mon.* injectargs \ + ceph tell 'osd.*' injectargs '--osd-beacon-report-interval 10' || exit 1 + ceph tell 'mon.*' injectargs \ '--mon-min-osdmap-epochs 2 --paxos-service-trim-min 1' || exit 1 create_pool foo 32 diff --git a/ceph/qa/standalone/osd/bad-inc-map.sh b/ceph/qa/standalone/osd/bad-inc-map.sh new file mode 100755 index 000000000..cc3cf27cc --- /dev/null +++ b/ceph/qa/standalone/osd/bad-inc-map.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +mon_port=$(get_unused_port) + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:$mon_port" + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + set -e + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_bad_inc_map() { + local dir=$1 + + run_mon $dir a + run_mgr $dir x + run_osd $dir 0 + run_osd $dir 1 + run_osd $dir 2 + + ceph config set osd.2 osd_inject_bad_map_crc_probability 1 + + # osd map churn + create_pool foo 8 + ceph osd pool set foo min_size 1 + ceph osd pool set foo min_size 2 + + sleep 5 + + # make sure all the OSDs are still up + TIMEOUT=10 wait_for_osd up 0 + TIMEOUT=10 wait_for_osd up 1 + TIMEOUT=10 wait_for_osd up 2 + + # check for the signature in the log + grep "injecting map crc failure" $dir/osd.2.log || return 1 + grep "bailing because last" $dir/osd.2.log || return 1 + + echo success + + delete_pool foo + kill_daemons $dir || return 1 +} + +main bad-inc-map "$@" + +# Local Variables: +# compile-command: "make -j4 && ../qa/run-standalone.sh bad-inc-map.sh" +# End: diff --git a/ceph/qa/standalone/osd/osd-backfill-stats.sh b/ceph/qa/standalone/osd/osd-backfill-stats.sh index db8f4da66..87c6218fe 100755 --- a/ceph/qa/standalone/osd/osd-backfill-stats.sh +++ b/ceph/qa/standalone/osd/osd-backfill-stats.sh @@ -152,8 +152,10 @@ function TEST_backfill_sizeup() { rados -p $poolname put obj$i /dev/null done + ceph osd set nobackfill ceph osd pool set $poolname size 3 - sleep 15 + sleep 2 + ceph osd unset nobackfill wait_for_clean || return 1 @@ -202,9 +204,11 @@ function TEST_backfill_sizeup_out() { # Remember primary during the backfill local primary=$(get_primary $poolname obj1) + ceph osd set nobackfill ceph osd out osd.$primary ceph osd pool set $poolname size 3 - sleep 15 + sleep 2 + ceph osd unset nobackfill wait_for_clean || return 1 @@ -249,8 +253,10 @@ function TEST_backfill_out() { # Remember primary during the backfill local primary=$(get_primary $poolname obj1) + ceph osd set nobackfill ceph osd out osd.$(get_not_primary $poolname obj1) - sleep 15 + sleep 2 + ceph osd unset nobackfill wait_for_clean || return 1 @@ -296,10 +302,12 @@ function TEST_backfill_down_out() { local primary=$(get_primary $poolname obj1) local otherosd=$(get_not_primary $poolname obj1) + ceph osd set nobackfill kill $(cat $dir/osd.${otherosd}.pid) ceph osd down osd.${otherosd} ceph osd out osd.${otherosd} - sleep 15 + sleep 2 + ceph osd unset nobackfill wait_for_clean || return 1 diff --git a/ceph/qa/standalone/osd/osd-rep-recov-eio.sh b/ceph/qa/standalone/osd/osd-rep-recov-eio.sh index 8dce41a98..613bfc316 100755 --- a/ceph/qa/standalone/osd/osd-rep-recov-eio.sh +++ b/ceph/qa/standalone/osd/osd-rep-recov-eio.sh @@ -19,6 +19,8 @@ source $CEPH_ROOT/qa/standalone/ceph-helpers.sh +warnings=10 + function run() { local dir=$1 shift @@ -32,7 +34,8 @@ function run() { local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do setup $dir || return 1 - run_mon $dir a || return 1 + # set warning amount in case default changes + run_mon $dir a --mon_osd_warn_num_repaired=$warnings || return 1 run_mgr $dir x || return 1 ceph osd pool create foo 8 || return 1 @@ -171,6 +174,86 @@ function TEST_rados_get_with_eio() { delete_pool $poolname } +function TEST_rados_repair_warning() { + local dir=$1 + local OBJS=$(expr $warnings + 1) + + setup_osds 4 || return 1 + + local poolname=pool-rep + create_pool $poolname 1 1 || return 1 + wait_for_clean || return 1 + + local poolname=pool-rep + local obj-base=obj-warn- + local inject=eio + + for i in $(seq 1 $OBJS) + do + rados_put $dir $poolname ${objbase}-$i || return 1 + inject_$inject rep data $poolname ${objbase}-$i $dir 0 || return 1 + rados_get $dir $poolname ${objbase}-$i || return 1 + done + local pgid=$(get_pg $poolname ${objbase}-1) + + local object_osds=($(get_osds $poolname ${objbase}-1)) + local primary=${object_osds[0]} + local bad_peer=${object_osds[1]} + + COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired') + test "$COUNT" = "$OBJS" || return 1 + flush_pg_stats + COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired") + test "$COUNT" = "$OBJS" || return 1 + + ceph health | grep -q "Too many repaired reads on 1 OSDs" || return 1 + ceph health detail | grep -q "osd.$primary had $OBJS reads repaired" || return 1 + + ceph health mute OSD_TOO_MANY_REPAIRS + set -o pipefail + # Should mute this + ceph health | $(! grep -q "Too many repaired reads on 1 OSDs") || return 1 + set +o pipefail + + for i in $(seq 1 $OBJS) + do + inject_$inject rep data $poolname ${objbase}-$i $dir 0 || return 1 + inject_$inject rep data $poolname ${objbase}-$i $dir 1 || return 1 + # Force primary to pull from the bad peer, so we can repair it too! + set_config osd $primary osd_debug_feed_pullee $bad_peer || return 1 + rados_get $dir $poolname ${objbase}-$i || return 1 + done + + COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired') + test "$COUNT" = "$(expr $OBJS \* 2)" || return 1 + flush_pg_stats + COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired") + test "$COUNT" = "$(expr $OBJS \* 3)" || return 1 + + # Give mon a chance to notice additional OSD and unmute + # The default tick time is 5 seconds + CHECKTIME=10 + LOOPS=0 + while(true) + do + sleep 1 + if ceph health | grep -q "Too many repaired reads on 2 OSDs" + then + break + fi + LOOPS=$(expr $LOOPS + 1) + if test "$LOOPS" = "$CHECKTIME" + then + echo "Too many repaired reads not seen after $CHECKTIME seconds" + return 1 + fi + done + ceph health detail | grep -q "osd.$primary had $(expr $OBJS \* 2) reads repaired" || return 1 + ceph health detail | grep -q "osd.$bad_peer had $OBJS reads repaired" || return 1 + + delete_pool $poolname +} + # Test backfill with unfound object function TEST_rep_backfill_unfound() { local dir=$1 diff --git a/ceph/qa/standalone/scrub/osd-scrub-test.sh b/ceph/qa/standalone/scrub/osd-scrub-test.sh index 39c95ff29..895349380 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-test.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-test.sh @@ -230,6 +230,120 @@ function TEST_scrub_extented_sleep() { teardown $dir || return 1 } +function _scrub_abort() { + local dir=$1 + local poolname=test + local OSDS=3 + local objects=1000 + local type=$2 + + TESTDATA="testdata.$$" + if test $type = "scrub"; + then + stopscrub="noscrub" + check="noscrub" + else + stopscrub="nodeep-scrub" + check="nodeep_scrub" + fi + + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=3 || return 1 + run_mgr $dir x || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd --osd_pool_default_pg_autoscale_mode=off \ + --osd_deep_scrub_randomize_ratio=0.0 \ + --osd_scrub_sleep=5.0 \ + --osd_scrub_interval_randomize_ratio=0 || return 1 + done + + # Create a pool with a single pg + create_pool $poolname 1 1 + wait_for_clean || return 1 + poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }') + + dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 + for i in `seq 1 $objects` + do + rados -p $poolname put obj${i} $TESTDATA + done + rm -f $TESTDATA + + local primary=$(get_primary $poolname obj1) + local pgid="${poolid}.0" + + ceph tell $pgid $type || return 1 + # deep-scrub won't start without scrub noticing + if [ "$type" = "deep_scrub" ]; + then + ceph tell $pgid scrub || return 1 + fi + + # Wait for scrubbing to start + set -o pipefail + found="no" + for i in $(seq 0 200) + do + flush_pg_stats + if ceph pg dump pgs | grep ^$pgid| grep -q "scrubbing" + then + found="yes" + #ceph pg dump pgs + break + fi + done + set +o pipefail + + if test $found = "no"; + then + echo "Scrubbing never started" + return 1 + fi + + ceph osd set $stopscrub + + # Wait for scrubbing to end + set -o pipefail + for i in $(seq 0 200) + do + flush_pg_stats + if ceph pg dump pgs | grep ^$pgid | grep -q "scrubbing" + then + continue + fi + #ceph pg dump pgs + break + done + set +o pipefail + + sleep 5 + + if ! grep "$check set, aborting" $dir/osd.${primary}.log + then + echo "Abort not seen in log" + return 1 + fi + + local last_scrub=$(get_last_scrub_stamp $pgid) + ceph osd unset noscrub + TIMEOUT=$(($objects / 2)) + wait_for_scrub $pgid "$last_scrub" || return 1 + + teardown $dir || return 1 +} + +function TEST_scrub_abort() { + local dir=$1 + _scrub_abort $dir scrub +} + +function TEST_deep_scrub_abort() { + local dir=$1 + _scrub_abort $dir deep_scrub +} + main osd-scrub-test "$@" # Local Variables: diff --git a/ceph/qa/standalone/special/ceph_objectstore_tool.py b/ceph/qa/standalone/special/ceph_objectstore_tool.py index 2790bc19b..527f2a7e1 100755 --- a/ceph/qa/standalone/special/ceph_objectstore_tool.py +++ b/ceph/qa/standalone/special/ceph_objectstore_tool.py @@ -1035,7 +1035,7 @@ def main(argv): # Specify a bad --op command cmd = (CFSD_PREFIX + "--op oops").format(osd=ONEOSD) - ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, dump-export, trim-pg-log)") + ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, dump-export, trim-pg-log, statfs)") # Provide just the object param not a command cmd = (CFSD_PREFIX + "object").format(osd=ONEOSD) diff --git a/ceph/qa/suites/krbd/fsx/conf.yaml b/ceph/qa/suites/krbd/fsx/conf.yaml index 30da870b2..d4863aa51 100644 --- a/ceph/qa/suites/krbd/fsx/conf.yaml +++ b/ceph/qa/suites/krbd/fsx/conf.yaml @@ -3,3 +3,5 @@ overrides: conf: global: ms die on skipped message: false + client: + rbd default map options: read_from_replica=balance diff --git a/ceph/qa/suites/multimds/basic/tasks/cephfs_test_exports.yaml b/ceph/qa/suites/multimds/basic/tasks/cephfs_test_exports.yaml index 46334fe16..6eb6c987c 100644 --- a/ceph/qa/suites/multimds/basic/tasks/cephfs_test_exports.yaml +++ b/ceph/qa/suites/multimds/basic/tasks/cephfs_test_exports.yaml @@ -1,3 +1,7 @@ +overrides: + ceph: + log-whitelist: + - Replacing daemon mds tasks: - cephfs_test_runner: fail_on_skip: false diff --git a/ceph/qa/suites/rados/cephadm/workunits/task/test_orch_cli.yaml b/ceph/qa/suites/rados/cephadm/workunits/task/test_orch_cli.yaml new file mode 100644 index 000000000..b62872123 --- /dev/null +++ b/ceph/qa/suites/rados/cephadm/workunits/task/test_orch_cli.yaml @@ -0,0 +1,18 @@ +roles: +- - host.a + - osd.0 + - osd.1 + - osd.2 + - mon.a + - mgr.a + - client.0 +tasks: +- install: +- cephadm: +- cephadm.shell: + host.a: + - ceph orch apply mds a +- cephfs_test_runner: + modules: + - tasks.cephfs.test_nfs + - tasks.cephadm_cases.test_cli diff --git a/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml b/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml index 9fa956b7e..11053d6a2 100644 --- a/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml +++ b/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml @@ -19,6 +19,7 @@ tasks: - \(MGR_ZABBIX_ - foo bar - Failed to open Telegraf + - evicting unresponsive client - cephfs_test_runner: modules: - tasks.mgr.test_module_selftest diff --git a/ceph/qa/suites/rados/singleton/all/random-eio.yaml b/ceph/qa/suites/rados/singleton/all/random-eio.yaml index fd1206805..1d5adae75 100644 --- a/ceph/qa/suites/rados/singleton/all/random-eio.yaml +++ b/ceph/qa/suites/rados/singleton/all/random-eio.yaml @@ -24,6 +24,7 @@ tasks: - overall HEALTH_ - \(POOL_APP_NOT_ENABLED\) - \(PG_DEGRADED\) + - \(OSD_TOO_MANY_REPAIRS\) - full_sequential: - exec: client.0: diff --git a/ceph/qa/suites/rados/thrash/crc-failures/bad_map_crc_failure.yaml b/ceph/qa/suites/rados/thrash/crc-failures/bad_map_crc_failure.yaml new file mode 100644 index 000000000..1e04fb369 --- /dev/null +++ b/ceph/qa/suites/rados/thrash/crc-failures/bad_map_crc_failure.yaml @@ -0,0 +1,7 @@ +overrides: + ceph: + conf: + osd: + osd inject bad map crc probability: 0.1 + log-whitelist: + - failed to encode map diff --git a/ceph/qa/suites/rados/thrash/crc-failures/default.yaml b/ceph/qa/suites/rados/thrash/crc-failures/default.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/.qa b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/.qa new file mode 120000 index 000000000..a23f7e045 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/.qa @@ -0,0 +1 @@ +../../.qa \ No newline at end of file diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/% b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/+ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml new file mode 100644 index 000000000..b0f3b9b4d --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml @@ -0,0 +1,4 @@ +openstack: + - volumes: # attached to each instance + count: 4 + size: 30 # GB diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml new file mode 100644 index 000000000..c631b0ed2 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml @@ -0,0 +1,21 @@ +meta: +- desc: | + Insatll and run ceph on one node, + with a separate client 1. + Upgrade client 1 to octopus + Run tests against old cluster +roles: +- - mon.a + - mon.b + - mon.c + - osd.0 + - osd.1 + - osd.2 + - client.0 + - mgr.x +- - client.1 +overrides: + ceph: + #log-whitelist: + #- failed to encode map + fs: xfs diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml new file mode 100644 index 000000000..e8758027f --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml @@ -0,0 +1,11 @@ +tasks: +- install: + branch: octopus + exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev','python34-cephfs','python34-rados'] +- print: "**** done install octopus" +- install.upgrade: + exclude_packages: ['ceph-test', 'ceph-test-dbg','libcephfs1', 'python-ceph'] + client.1: +- print: "**** done install.upgrade to -x on client.0" +- ceph: +- print: "**** done ceph task" diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml new file mode 100644 index 000000000..dff6623ad --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + rbd default features: 61 + diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml new file mode 100644 index 000000000..5613d0155 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + rbd default features: 1 + diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml new file mode 100644 index 000000000..10b86530a --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml @@ -0,0 +1,36 @@ +tasks: +- parallel: + - workunit: + branch: octopus + clients: + client.0: + - rbd/notify_master.sh + env: + RBD_FEATURES: "61" + - workunit: + #The line below to change to 'pacific' + branch: master + clients: + client.1: + - rbd/notify_slave.sh + env: + RBD_FEATURES: "61" +- print: "**** done rbd: old librbd -> new librbd" +- parallel: + - workunit: + #The line below to change to 'pacific' + branch: master + clients: + client.0: + - rbd/notify_slave.sh + env: + RBD_FEATURES: "61" + - workunit: + #The line below to change to 'pacific' + branch: master + clients: + client.1: + - rbd/notify_master.sh + env: + RBD_FEATURES: "61" +- print: "**** done rbd: new librbd -> old librbd" diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml new file mode 120000 index 000000000..886e87fa2 --- /dev/null +++ b/ceph/qa/suites/upgrade-clients/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/ubuntu_18.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/.qa deleted file mode 120000 index a602a0353..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/% b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/% deleted file mode 100644 index e69de29bb..000000000 diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa deleted file mode 120000 index a602a0353..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/+ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/+ deleted file mode 100644 index e69de29bb..000000000 diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa deleted file mode 120000 index a602a0353..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml deleted file mode 100644 index b0f3b9b4d..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml +++ /dev/null @@ -1,4 +0,0 @@ -openstack: - - volumes: # attached to each instance - count: 4 - size: 30 # GB diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml deleted file mode 100644 index c631b0ed2..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml +++ /dev/null @@ -1,21 +0,0 @@ -meta: -- desc: | - Insatll and run ceph on one node, - with a separate client 1. - Upgrade client 1 to octopus - Run tests against old cluster -roles: -- - mon.a - - mon.b - - mon.c - - osd.0 - - osd.1 - - osd.2 - - client.0 - - mgr.x -- - client.1 -overrides: - ceph: - #log-whitelist: - #- failed to encode map - fs: xfs diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa deleted file mode 120000 index a602a0353..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml deleted file mode 100644 index e8758027f..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml +++ /dev/null @@ -1,11 +0,0 @@ -tasks: -- install: - branch: octopus - exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev','python34-cephfs','python34-rados'] -- print: "**** done install octopus" -- install.upgrade: - exclude_packages: ['ceph-test', 'ceph-test-dbg','libcephfs1', 'python-ceph'] - client.1: -- print: "**** done install.upgrade to -x on client.0" -- ceph: -- print: "**** done ceph task" diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa deleted file mode 120000 index a602a0353..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml deleted file mode 100644 index dff6623ad..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml +++ /dev/null @@ -1,6 +0,0 @@ -overrides: - ceph: - conf: - client: - rbd default features: 61 - diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml deleted file mode 100644 index 5613d0155..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml +++ /dev/null @@ -1,6 +0,0 @@ -overrides: - ceph: - conf: - client: - rbd default features: 1 - diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa deleted file mode 120000 index a602a0353..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml deleted file mode 100644 index 10b86530a..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml +++ /dev/null @@ -1,36 +0,0 @@ -tasks: -- parallel: - - workunit: - branch: octopus - clients: - client.0: - - rbd/notify_master.sh - env: - RBD_FEATURES: "61" - - workunit: - #The line below to change to 'pacific' - branch: master - clients: - client.1: - - rbd/notify_slave.sh - env: - RBD_FEATURES: "61" -- print: "**** done rbd: old librbd -> new librbd" -- parallel: - - workunit: - #The line below to change to 'pacific' - branch: master - clients: - client.0: - - rbd/notify_slave.sh - env: - RBD_FEATURES: "61" - - workunit: - #The line below to change to 'pacific' - branch: master - clients: - client.1: - - rbd/notify_master.sh - env: - RBD_FEATURES: "61" -- print: "**** done rbd: new librbd -> old librbd" diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa deleted file mode 120000 index a602a0353..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml deleted file mode 120000 index 886e87fa2..000000000 --- a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../../../distros/all/ubuntu_18.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml index b03f5fab7..8f9b961ca 100644 --- a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml @@ -3,7 +3,7 @@ meta: librbd python api tests tasks: - workunit: - tag: v15.2.1 + tag: v15.2.4 clients: client.0: - rbd/test_librbd_python.sh diff --git a/ceph/qa/tasks/ceph_manager.py b/ceph/qa/tasks/ceph_manager.py index c058735aa..251604321 100644 --- a/ceph/qa/tasks/ceph_manager.py +++ b/ceph/qa/tasks/ceph_manager.py @@ -2467,6 +2467,36 @@ class CephManager: pgs = self.get_pg_stats() return self._get_num_active_down(pgs) == len(pgs) + def dump_pgs_not_active_clean(self): + """ + Dumps all pgs that are not active+clean + """ + pgs = self.get_pg_stats() + for pg in pgs: + if pg['state'] != 'active+clean': + self.log('PG %s is not active+clean' % pg['pgid']) + self.log(pg) + + def dump_pgs_not_active_down(self): + """ + Dumps all pgs that are not active or down + """ + pgs = self.get_pg_stats() + for pg in pgs: + if 'active' not in pg['state'] and 'down' not in pg['state']: + self.log('PG %s is not active or down' % pg['pgid']) + self.log(pg) + + def dump_pgs_not_active(self): + """ + Dumps all pgs that are not active + """ + pgs = self.get_pg_stats() + for pg in pgs: + if 'active' not in pg['state']: + self.log('PG %s is not active' % pg['pgid']) + self.log(pg) + def wait_for_clean(self, timeout=1200): """ Returns true when all pgs are clean. @@ -2482,11 +2512,10 @@ class CephManager: else: self.log("no progress seen, keeping timeout for now") if time.time() - start >= timeout: - self.log('dumping pgs') - out = self.raw_cluster_cmd('pg', 'dump') - self.log(out) + self.log('dumping pgs not clean') + self.dump_pgs_not_active_clean() assert time.time() - start < timeout, \ - 'failed to become clean before timeout expired' + 'wait_for_clean: failed before timeout expired' cur_active_clean = self.get_num_active_clean() if cur_active_clean != num_active_clean: start = time.time() @@ -2568,11 +2597,10 @@ class CephManager: if now - start >= timeout: if self.is_recovered(): break - self.log('dumping pgs') - out = self.raw_cluster_cmd('pg', 'dump') - self.log(out) + self.log('dumping pgs not recovered yet') + self.dump_pgs_not_active_clean() assert now - start < timeout, \ - 'failed to recover before timeout expired' + 'wait_for_recovery: failed before timeout expired' cur_active_recovered = self.get_num_active_recovered() if cur_active_recovered != num_active_recovered: start = time.time() @@ -2590,11 +2618,10 @@ class CephManager: while not self.is_active(): if timeout is not None: if time.time() - start >= timeout: - self.log('dumping pgs') - out = self.raw_cluster_cmd('pg', 'dump') - self.log(out) + self.log('dumping pgs not active') + self.dump_pgs_not_active() assert time.time() - start < timeout, \ - 'failed to recover before timeout expired' + 'wait_for_active: failed before timeout expired' cur_active = self.get_num_active() if cur_active != num_active: start = time.time() @@ -2613,11 +2640,10 @@ class CephManager: while not self.is_active_or_down(): if timeout is not None: if time.time() - start >= timeout: - self.log('dumping pgs') - out = self.raw_cluster_cmd('pg', 'dump') - self.log(out) + self.log('dumping pgs not active or down') + self.dump_pgs_not_active_down() assert time.time() - start < timeout, \ - 'failed to recover before timeout expired' + 'wait_for_active_or_down: failed before timeout expired' cur_active_down = self.get_num_active_down() if cur_active_down != num_active_down: start = time.time() @@ -2667,11 +2693,10 @@ class CephManager: while not self.is_active(): if timeout is not None: if time.time() - start >= timeout: - self.log('dumping pgs') - out = self.raw_cluster_cmd('pg', 'dump') - self.log(out) + self.log('dumping pgs not active') + self.dump_pgs_not_active() assert time.time() - start < timeout, \ - 'failed to become active before timeout expired' + 'wait_till_active: failed before timeout expired' time.sleep(3) self.log("active!") diff --git a/ceph/qa/tasks/cephadm.py b/ceph/qa/tasks/cephadm.py index a5065ba3e..42b6f3e98 100644 --- a/ceph/qa/tasks/cephadm.py +++ b/ceph/qa/tasks/cephadm.py @@ -1150,20 +1150,25 @@ def task(ctx, config): container_registry_mirror = mirrors.get('docker.io', container_registry_mirror) - if not container_image_name: - raise Exception("Configuration error occurred. " - "The 'image' value is undefined for 'cephadm' task. " - "Please provide corresponding options in the task's " - "config, task 'overrides', or teuthology 'defaults' " - "section.") if not hasattr(ctx.ceph[cluster_name], 'image'): ctx.ceph[cluster_name].image = config.get('image') ref = None if not ctx.ceph[cluster_name].image: + if not container_image_name: + raise Exception("Configuration error occurred. " + "The 'image' value is undefined for 'cephadm' task. " + "Please provide corresponding options in the task's " + "config, task 'overrides', or teuthology 'defaults' " + "section.") sha1 = config.get('sha1') + flavor = config.get('flavor', 'default') + if sha1: - ctx.ceph[cluster_name].image = container_image_name + ':' + sha1 + if flavor == "crimson": + ctx.ceph[cluster_name].image = container_image_name + ':' + sha1 + '-' + flavor + else: + ctx.ceph[cluster_name].image = container_image_name + ':' + sha1 ref = sha1 else: # hmm, fall back to branch? diff --git a/ceph/qa/tasks/cephadm_cases/__init__.py b/ceph/qa/tasks/cephadm_cases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/tasks/cephadm_cases/test_cli.py b/ceph/qa/tasks/cephadm_cases/test_cli.py new file mode 100644 index 000000000..f6a7521b6 --- /dev/null +++ b/ceph/qa/tasks/cephadm_cases/test_cli.py @@ -0,0 +1,52 @@ +import logging + +from tasks.mgr.mgr_test_case import MgrTestCase + +log = logging.getLogger(__name__) + + +class TestCephadmCLI(MgrTestCase): + def _cmd(self, *args): + return self.mgr_cluster.mon_manager.raw_cluster_cmd(*args) + + def _orch_cmd(self, *args): + return self._cmd("orch", *args) + + def setUp(self): + super(TestCephadmCLI, self).setUp() + + def test_yaml(self): + """ + to prevent oddities like + + >>> import yaml + ... from collections import OrderedDict + ... assert yaml.dump(OrderedDict()) == '!!python/object/apply:collections.OrderedDict\\n- []\\n' + """ + out = self._orch_cmd('device', 'ls', '--format', 'yaml') + self.assertNotIn('!!python', out) + + out = self._orch_cmd('host', 'ls', '--format', 'yaml') + self.assertNotIn('!!python', out) + + out = self._orch_cmd('ls', '--format', 'yaml') + self.assertNotIn('!!python', out) + + out = self._orch_cmd('ps', '--format', 'yaml') + self.assertNotIn('!!python', out) + + out = self._orch_cmd('status', '--format', 'yaml') + self.assertNotIn('!!python', out) + + def test_pause(self): + self._orch_cmd('pause') + self.wait_for_health('CEPHADM_PAUSED', 30) + self._orch_cmd('resume') + self.wait_for_health_clear(30) + + def test_daemon_restart(self): + self._orch_cmd('daemon', 'stop', 'osd.0') + self.wait_for_health('OSD_DOWN', 30) + self._orch_cmd('daemon', 'start', 'osd.0') + self.wait_for_health_clear(30) + self._orch_cmd('daemon', 'restart', 'osd.0') diff --git a/ceph/qa/tasks/cephfs/cephfs_test_case.py b/ceph/qa/tasks/cephfs/cephfs_test_case.py index e69941dfd..063a6508c 100644 --- a/ceph/qa/tasks/cephfs/cephfs_test_case.py +++ b/ceph/qa/tasks/cephfs/cephfs_test_case.py @@ -1,4 +1,3 @@ -import time import json import logging from tasks.ceph_test_case import CephTestCase @@ -7,6 +6,7 @@ import re from tasks.cephfs.fuse_mount import FuseMount +from teuthology import contextutil from teuthology.orchestra import run from teuthology.orchestra.run import CommandFailedError from teuthology.contextutil import safe_while @@ -265,6 +265,10 @@ class CephFSTestCase(CephTestCase): if core_dir: # Non-default core_pattern with a directory in it # We have seen a core_pattern that looks like it's from teuthology's coredump # task, so proceed to clear out the core file + if core_dir[0] == '|': + log.info("Piped core dumps to program {0}, skip cleaning".format(core_dir[1:])) + return; + log.info("Clearing core from directory: {0}".format(core_dir)) # Verify that we see the expected single coredump @@ -287,22 +291,50 @@ class CephFSTestCase(CephTestCase): else: log.info("No core_pattern directory set, nothing to clear (internal.coredump not enabled?)") - def _wait_subtrees(self, status, rank, test): - timeout = 30 - pause = 2 + def _get_subtrees(self, status=None, rank=None, path=None): + if path is None: + path = "/" + try: + with contextutil.safe_while(sleep=1, tries=3) as proceed: + while proceed(): + try: + if rank == "all": + subtrees = [] + for r in self.fs.get_ranks(status=status): + s = self.fs.rank_asok(["get", "subtrees"], status=status, rank=r['rank']) + s = filter(lambda s: s['auth_first'] == r['rank'] and s['auth_second'] == -2, s) + subtrees += s + else: + subtrees = self.fs.rank_asok(["get", "subtrees"], status=status, rank=rank) + subtrees = filter(lambda s: s['dir']['path'].startswith(path), subtrees) + return list(subtrees) + except CommandFailedError as e: + # Sometimes we get transient errors + if e.exitstatus == 22: + pass + else: + raise + except contextutil.MaxWhileTries as e: + raise RuntimeError(f"could not get subtree state from rank {rank}") from e + + def _wait_subtrees(self, test, status=None, rank=None, timeout=30, sleep=2, action=None, path=None): test = sorted(test) - for i in range(timeout // pause): - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, rank)['name']) - subtrees = filter(lambda s: s['dir']['path'].startswith('/'), subtrees) - filtered = sorted([(s['dir']['path'], s['auth_first']) for s in subtrees]) - log.info("%s =?= %s", filtered, test) - if filtered == test: - # Confirm export_pin in output is correct: - for s in subtrees: - self.assertTrue(s['export_pin'] == s['auth_first']) - return subtrees - time.sleep(pause) - raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) + try: + with contextutil.safe_while(sleep=sleep, tries=timeout//sleep) as proceed: + while proceed(): + subtrees = self._get_subtrees(status=status, rank=rank, path=path) + filtered = sorted([(s['dir']['path'], s['auth_first']) for s in subtrees]) + log.info("%s =?= %s", filtered, test) + if filtered == test: + # Confirm export_pin in output is correct: + for s in subtrees: + if s['export_pin'] >= 0: + self.assertTrue(s['export_pin'] == s['auth_first']) + return subtrees + if action is not None: + action() + except contextutil.MaxWhileTries as e: + raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) from e def _wait_until_scrub_complete(self, path="/", recursive=True): out_json = self.fs.rank_tell(["scrub", "start", path] + ["recursive"] if recursive else []) @@ -312,3 +344,26 @@ class CephFSTestCase(CephTestCase): if out_json['status'] == "no active scrubs running": break; + def _wait_distributed_subtrees(self, count, status=None, rank=None, path=None): + try: + with contextutil.safe_while(sleep=5, tries=20) as proceed: + while proceed(): + subtrees = self._get_subtrees(status=status, rank=rank, path=path) + subtrees = list(filter(lambda s: s['distributed_ephemeral_pin'] == True, subtrees)) + log.info(f"len={len(subtrees)} {subtrees}") + if len(subtrees) >= count: + return subtrees + except contextutil.MaxWhileTries as e: + raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) from e + + def _wait_random_subtrees(self, count, status=None, rank=None, path=None): + try: + with contextutil.safe_while(sleep=5, tries=20) as proceed: + while proceed(): + subtrees = self._get_subtrees(status=status, rank=rank, path=path) + subtrees = list(filter(lambda s: s['random_ephemeral_pin'] == True, subtrees)) + log.info(f"len={len(subtrees)} {subtrees}") + if len(subtrees) >= count: + return subtrees + except contextutil.MaxWhileTries as e: + raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) from e diff --git a/ceph/qa/tasks/cephfs/filesystem.py b/ceph/qa/tasks/cephfs/filesystem.py index 2210fa970..0a9066fb6 100644 --- a/ceph/qa/tasks/cephfs/filesystem.py +++ b/ceph/qa/tasks/cephfs/filesystem.py @@ -223,12 +223,16 @@ class CephCluster(object): def json_asok(self, command, service_type, service_id, timeout=None): if timeout is None: timeout = 15*60 + command.insert(0, '--format=json') proc = self.mon_manager.admin_socket(service_type, service_id, command, timeout=timeout) - response_data = proc.stdout.getvalue() - log.info("_json_asok output: {0}".format(response_data)) - if response_data.strip(): - return json.loads(response_data) + response_data = proc.stdout.getvalue().strip() + if len(response_data) > 0: + j = json.loads(response_data) + pretty = json.dumps(j, sort_keys=True, indent=2) + log.debug(f"_json_asok output\n{pretty}") + return j else: + log.debug("_json_asok output empty") return None @@ -840,9 +844,11 @@ class Filesystem(MDSCluster): return result - def get_rank(self, rank=0, status=None): + def get_rank(self, rank=None, status=None): if status is None: status = self.getinfo() + if rank is None: + rank = 0 return status.get_rank(self.id, rank) def rank_restart(self, rank=0, status=None): @@ -1012,6 +1018,22 @@ class Filesystem(MDSCluster): info = self.get_rank(rank=rank, status=status) return json.loads(self.mon_manager.raw_cluster_cmd("tell", 'mds.{0}'.format(info['name']), *command)) + def ranks_tell(self, command, status=None): + if status is None: + status = self.status() + out = [] + for r in status.get_ranks(self.id): + result = self.rank_tell(command, rank=r['rank'], status=status) + out.append((r['rank'], result)) + return sorted(out) + + def ranks_perf(self, f, status=None): + perf = self.ranks_tell(["perf", "dump"], status=status) + out = [] + for rank, perf in perf: + out.append((rank, f(perf))) + return out + def read_cache(self, path, depth=None): cmd = ["dump", "tree", path] if depth is not None: diff --git a/ceph/qa/tasks/cephfs/mount.py b/ceph/qa/tasks/cephfs/mount.py index 7d04535c8..48f065051 100644 --- a/ceph/qa/tasks/cephfs/mount.py +++ b/ceph/qa/tasks/cephfs/mount.py @@ -8,7 +8,7 @@ from six import StringIO from textwrap import dedent import os from teuthology.orchestra import run -from teuthology.orchestra.run import CommandFailedError, ConnectionLostError +from teuthology.orchestra.run import CommandFailedError, ConnectionLostError, Raw from tasks.cephfs.filesystem import Filesystem log = logging.getLogger(__name__) @@ -197,6 +197,9 @@ class CephFSMount(object): p.wait() return six.ensure_str(p.stdout.getvalue().strip()) + def run_shell_payload(self, payload, **kwargs): + return self.run_shell(["bash", "-c", Raw(f"'{payload}'")], **kwargs) + def run_shell(self, args, wait=True, stdin=None, check_status=True, omit_sudo=True): if isinstance(args, str): diff --git a/ceph/qa/tasks/cephfs/test_client_recovery.py b/ceph/qa/tasks/cephfs/test_client_recovery.py index 9f8aa5dec..12ee909a6 100644 --- a/ceph/qa/tasks/cephfs/test_client_recovery.py +++ b/ceph/qa/tasks/cephfs/test_client_recovery.py @@ -636,7 +636,7 @@ class TestClientRecovery(CephFSTestCase): self.mount_a.umount_wait() if isinstance(self.mount_a, FuseMount): - self.skipTest("Not implemented in FUSE client yet") + self.mount_a.mount(mount_options=['--client_reconnect_stale=1', '--fuse_disable_pagecache=1']) else: try: self.mount_a.mount(mount_options=['recover_session=clean']) diff --git a/ceph/qa/tasks/cephfs/test_exports.py b/ceph/qa/tasks/cephfs/test_exports.py index abaf92e6b..3cced538d 100644 --- a/ceph/qa/tasks/cephfs/test_exports.py +++ b/ceph/qa/tasks/cephfs/test_exports.py @@ -1,7 +1,9 @@ import logging +import random import time from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase +from teuthology.orchestra.run import CommandFailedError log = logging.getLogger(__name__) @@ -9,123 +11,126 @@ class TestExports(CephFSTestCase): MDSS_REQUIRED = 2 CLIENTS_REQUIRED = 2 - def test_export_pin(self): + def test_session_race(self): + """ + Test session creation race. + + See: https://tracker.ceph.com/issues/24072#change-113056 + """ + self.fs.set_max_mds(2) - self.fs.wait_for_daemons() + status = self.fs.wait_for_daemons() - status = self.fs.status() + rank1 = self.fs.get_rank(rank=1, status=status) - self.mount_a.run_shell(["mkdir", "-p", "1/2/3"]) - self._wait_subtrees(status, 0, []) + # Create a directory that is pre-exported to rank 1 + self.mount_a.run_shell(["mkdir", "-p", "a/aa"]) + self.mount_a.setfattr("a", "ceph.dir.pin", "1") + self._wait_subtrees([('/a', 1)], status=status, rank=1) - # NOP - self.mount_a.setfattr("1", "ceph.dir.pin", "-1") - self._wait_subtrees(status, 0, []) + # Now set the mds config to allow the race + self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "true"], rank=1) - # NOP (rank < -1) - self.mount_a.setfattr("1", "ceph.dir.pin", "-2341") - self._wait_subtrees(status, 0, []) + # Now create another directory and try to export it + self.mount_b.run_shell(["mkdir", "-p", "b/bb"]) + self.mount_b.setfattr("b", "ceph.dir.pin", "1") - # pin /1 to rank 1 - self.mount_a.setfattr("1", "ceph.dir.pin", "1") - self._wait_subtrees(status, 1, [('/1', 1)]) + time.sleep(5) - # Check export_targets is set properly - status = self.fs.status() - log.info(status) - r0 = status.get_rank(self.fs.id, 0) - self.assertTrue(sorted(r0['export_targets']) == [1]) + # Now turn off the race so that it doesn't wait again + self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "false"], rank=1) - # redundant pin /1/2 to rank 1 - self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") - self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 1)]) + # Now try to create a session with rank 1 by accessing a dir known to + # be there, if buggy, this should cause the rank 1 to crash: + self.mount_b.run_shell(["ls", "a"]) - # change pin /1/2 to rank 0 - self.mount_a.setfattr("1/2", "ceph.dir.pin", "0") - self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 0)]) - self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0)]) + # Check if rank1 changed (standby tookover?) + new_rank1 = self.fs.get_rank(rank=1) + self.assertEqual(rank1['gid'], new_rank1['gid']) - # change pin /1/2/3 to (presently) non-existent rank 2 - self.mount_a.setfattr("1/2/3", "ceph.dir.pin", "2") - self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0)]) - self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 0)]) +class TestExportPin(CephFSTestCase): + MDSS_REQUIRED = 3 + CLIENTS_REQUIRED = 1 - # change pin /1/2 back to rank 1 - self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") - self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 1)]) + def setUp(self): + CephFSTestCase.setUp(self) - # add another directory pinned to 1 - self.mount_a.run_shell(["mkdir", "-p", "1/4/5"]) - self.mount_a.setfattr("1/4/5", "ceph.dir.pin", "1") - self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 1), ('/1/4/5', 1)]) + self.fs.set_max_mds(3) + self.status = self.fs.wait_for_daemons() - # change pin /1 to 0 - self.mount_a.setfattr("1", "ceph.dir.pin", "0") - self._wait_subtrees(status, 0, [('/1', 0), ('/1/2', 1), ('/1/4/5', 1)]) + self.mount_a.run_shell_payload("mkdir -p 1/2/3/4") - # change pin /1/2 to default (-1); does the subtree root properly respect it's parent pin? - self.mount_a.setfattr("1/2", "ceph.dir.pin", "-1") - self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1)]) - - if len(list(status.get_standbys())): - self.fs.set_max_mds(3) - self.fs.wait_for_state('up:active', rank=2) - self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2)]) - - # Check export_targets is set properly - status = self.fs.status() - log.info(status) - r0 = status.get_rank(self.fs.id, 0) - self.assertTrue(sorted(r0['export_targets']) == [1,2]) - r1 = status.get_rank(self.fs.id, 1) - self.assertTrue(sorted(r1['export_targets']) == [0]) - r2 = status.get_rank(self.fs.id, 2) - self.assertTrue(sorted(r2['export_targets']) == []) - - # Test rename - self.mount_a.run_shell(["mkdir", "-p", "a/b", "aa/bb"]) - self.mount_a.setfattr("a", "ceph.dir.pin", "1") - self.mount_a.setfattr("aa/bb", "ceph.dir.pin", "0") - if (len(self.fs.get_active_names()) > 2): - self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/aa/bb', 0)]) - else: - self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/a', 1), ('/aa/bb', 0)]) - self.mount_a.run_shell(["mv", "aa", "a/b/"]) - if (len(self.fs.get_active_names()) > 2): - self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/a/b/aa/bb', 0)]) - else: - self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/a', 1), ('/a/b/aa/bb', 0)]) + def test_noop(self): + self.mount_a.setfattr("1", "ceph.dir.pin", "-1") + time.sleep(30) # for something to not happen + self._wait_subtrees([], status=self.status) - def test_export_pin_getfattr(self): - self.fs.set_max_mds(2) - self.fs.wait_for_daemons() + def test_negative(self): + self.mount_a.setfattr("1", "ceph.dir.pin", "-2341") + time.sleep(30) # for something to not happen + self._wait_subtrees([], status=self.status) - status = self.fs.status() + def test_empty_pin(self): + self.mount_a.setfattr("1/2/3/4", "ceph.dir.pin", "1") + time.sleep(30) # for something to not happen + self._wait_subtrees([], status=self.status) - self.mount_a.run_shell(["mkdir", "-p", "1/2/3"]) - self._wait_subtrees(status, 0, []) + def test_trivial(self): + self.mount_a.setfattr("1", "ceph.dir.pin", "1") + self._wait_subtrees([('/1', 1)], status=self.status, rank=1) - # pin /1 to rank 0 + def test_export_targets(self): self.mount_a.setfattr("1", "ceph.dir.pin", "1") - self._wait_subtrees(status, 1, [('/1', 1)]) + self._wait_subtrees([('/1', 1)], status=self.status, rank=1) + self.status = self.fs.status() + r0 = self.status.get_rank(self.fs.id, 0) + self.assertTrue(sorted(r0['export_targets']) == [1]) - # pin /1/2 to rank 1 + def test_redundant(self): + # redundant pin /1/2 to rank 1 + self.mount_a.setfattr("1", "ceph.dir.pin", "1") + self._wait_subtrees([('/1', 1)], status=self.status, rank=1) self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") - self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 1)]) + self._wait_subtrees([('/1', 1), ('/1/2', 1)], status=self.status, rank=1) - # change pin /1/2 to rank 0 + def test_reassignment(self): + self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") + self._wait_subtrees([('/1/2', 1)], status=self.status, rank=1) self.mount_a.setfattr("1/2", "ceph.dir.pin", "0") - self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 0)]) - self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0)]) + self._wait_subtrees([('/1/2', 0)], status=self.status, rank=0) - # change pin /1/2/3 to (presently) non-existent rank 2 + def test_phantom_rank(self): + self.mount_a.setfattr("1", "ceph.dir.pin", "0") + self.mount_a.setfattr("1/2", "ceph.dir.pin", "10") + time.sleep(30) # wait for nothing weird to happen + self._wait_subtrees([('/1', 0)], status=self.status) + + def test_nested(self): + self.mount_a.setfattr("1", "ceph.dir.pin", "1") + self.mount_a.setfattr("1/2", "ceph.dir.pin", "0") self.mount_a.setfattr("1/2/3", "ceph.dir.pin", "2") - self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0)]) + self._wait_subtrees([('/1', 1), ('/1/2', 0), ('/1/2/3', 2)], status=self.status, rank=2) + + def test_nested_unset(self): + self.mount_a.setfattr("1", "ceph.dir.pin", "1") + self.mount_a.setfattr("1/2", "ceph.dir.pin", "2") + self._wait_subtrees([('/1', 1), ('/1/2', 2)], status=self.status, rank=1) + self.mount_a.setfattr("1/2", "ceph.dir.pin", "-1") + self._wait_subtrees([('/1', 1)], status=self.status, rank=1) + + def test_rename(self): + self.mount_a.setfattr("1", "ceph.dir.pin", "1") + self.mount_a.run_shell_payload("mkdir -p 9/8/7") + self.mount_a.setfattr("9/8", "ceph.dir.pin", "0") + self._wait_subtrees([('/1', 1), ("/9/8", 0)], status=self.status, rank=0) + self.mount_a.run_shell_payload("mv 9/8 1/2") + self._wait_subtrees([('/1', 1), ("/1/2/8", 0)], status=self.status, rank=0) - if len(list(status.get_standbys())): - self.fs.set_max_mds(3) - self.fs.wait_for_state('up:active', rank=2) - self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0), ('/1/2/3', 2)]) + def test_getfattr(self): + # pin /1 to rank 0 + self.mount_a.setfattr("1", "ceph.dir.pin", "1") + self.mount_a.setfattr("1/2", "ceph.dir.pin", "0") + self._wait_subtrees([('/1', 1), ('/1/2', 0)], status=self.status, rank=1) if not isinstance(self.mount_a, FuseMount): p = self.mount_a.client_remote.sh('uname -r', wait=True) @@ -135,42 +140,373 @@ class TestExports(CephFSTestCase): self.skipTest("Kernel does not support getting the extended attribute ceph.dir.pin") self.assertEqual(self.mount_a.getfattr("1", "ceph.dir.pin"), '1') self.assertEqual(self.mount_a.getfattr("1/2", "ceph.dir.pin"), '0') - if (len(self.fs.get_active_names()) > 2): - self.assertEqual(self.mount_a.getfattr("1/2/3", "ceph.dir.pin"), '2') - def test_session_race(self): + def test_export_pin_cache_drop(self): + """ + That the export pin does not prevent empty (nothing in cache) subtree merging. """ - Test session creation race. - See: https://tracker.ceph.com/issues/24072#change-113056 + self.mount_a.setfattr("1", "ceph.dir.pin", "0") + self.mount_a.setfattr("1/2", "ceph.dir.pin", "1") + self._wait_subtrees([('/1', 0), ('/1/2', 1)], status=self.status) + self.mount_a.umount_wait() # release all caps + def _drop(): + self.fs.ranks_tell(["cache", "drop"], status=self.status) + # drop cache multiple times to clear replica pins + self._wait_subtrees([], status=self.status, action=_drop) + +class TestEphemeralPins(CephFSTestCase): + MDSS_REQUIRED = 3 + CLIENTS_REQUIRED = 1 + + def setUp(self): + CephFSTestCase.setUp(self) + + self.config_set('mds', 'mds_export_ephemeral_random', True) + self.config_set('mds', 'mds_export_ephemeral_distributed', True) + self.config_set('mds', 'mds_export_ephemeral_random_max', 1.0) + + self.mount_a.run_shell_payload(""" +set -e + +# Use up a random number of inode numbers so the ephemeral pinning is not the same every test. +mkdir .inode_number_thrash +count=$((RANDOM % 1024)) +for ((i = 0; i < count; i++)); do touch .inode_number_thrash/$i; done +rm -rf .inode_number_thrash +""") + + self.fs.set_max_mds(3) + self.status = self.fs.wait_for_daemons() + + def _setup_tree(self, path="tree", export=-1, distributed=False, random=0.0, count=100, wait=True): + return self.mount_a.run_shell_payload(f""" +set -e +mkdir -p {path} +{f"setfattr -n ceph.dir.pin -v {export} {path}" if export >= 0 else ""} +{f"setfattr -n ceph.dir.pin.distributed -v 1 {path}" if distributed else ""} +{f"setfattr -n ceph.dir.pin.random -v {random} {path}" if random > 0.0 else ""} +for ((i = 0; i < {count}; i++)); do + mkdir -p "{path}/$i" + echo file > "{path}/$i/file" +done +""", wait=wait) + + def test_ephemeral_pin_dist_override(self): + """ + That an ephemeral distributed pin overrides a normal export pin. """ - self.fs.set_max_mds(2) - status = self.fs.wait_for_daemons() + self._setup_tree(distributed=True) + subtrees = self._wait_distributed_subtrees(100, status=self.status, rank="all") + for s in subtrees: + path = s['dir']['path'] + if path == '/tree': + self.assertEqual(s['export_pin'], 0) + self.assertEqual(s['auth_first'], 0) + elif path.startswith('/tree/'): + self.assertEqual(s['export_pin'], -1) + self.assertTrue(s['distributed_ephemeral_pin']) + + def test_ephemeral_pin_dist_override_pin(self): + """ + That an export pin overrides an ephemerally pinned directory. + """ - rank1 = self.fs.get_rank(rank=1, status=status) + self._setup_tree(distributed=True, export=0) + subtrees = self._wait_distributed_subtrees(100, status=self.status, rank="all", path="/tree/") + which = None + for s in subtrees: + if s['auth_first'] == 1: + path = s['dir']['path'] + self.mount_a.setfattr(path[1:], "ceph.dir.pin", "0") + which = path + break + self.assertIsNotNone(which) + time.sleep(15) + subtrees = self._get_subtrees(status=self.status, rank=0) + for s in subtrees: + path = s['dir']['path'] + if path == which: + self.assertEqual(s['auth_first'], 0) + self.assertFalse(s['distributed_ephemeral_pin']) + return + # it has been merged into /tree + + def test_ephemeral_pin_dist_off(self): + """ + That turning off ephemeral distributed pin merges subtrees. + """ - # Create a directory that is pre-exported to rank 1 - self.mount_a.run_shell(["mkdir", "-p", "a/aa"]) - self.mount_a.setfattr("a", "ceph.dir.pin", "1") - self._wait_subtrees(status, 1, [('/a', 1)]) + self._setup_tree(distributed=True, export=0) + self._wait_distributed_subtrees(100, status=self.status, rank="all") + self.mount_a.setfattr("tree", "ceph.dir.pin.distributed", "0") + self._wait_subtrees([('/tree', 0)], status=self.status) - # Now set the mds config to allow the race - self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "true"], rank=1) + def test_ephemeral_pin_dist_conf_off(self): + """ + That turning off ephemeral distributed pin config prevents distribution. + """ - # Now create another directory and try to export it - self.mount_b.run_shell(["mkdir", "-p", "b/bb"]) - self.mount_b.setfattr("b", "ceph.dir.pin", "1") + self._setup_tree(export=0) + self.config_set('mds', 'mds_export_ephemeral_distributed', False) + self.mount_a.setfattr("tree", "ceph.dir.pin.distributed", "1") + time.sleep(30) + self._wait_subtrees([('/tree', 0)], status=self.status) - time.sleep(5) + def test_ephemeral_pin_dist_conf_off_merge(self): + """ + That turning off ephemeral distributed pin config merges subtrees. + """ - # Now turn off the race so that it doesn't wait again - self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "false"], rank=1) + self._setup_tree(distributed=True, export=0) + self._wait_distributed_subtrees(100, status=self.status) + self.config_set('mds', 'mds_export_ephemeral_distributed', False) + self._wait_subtrees([('/tree', 0)], timeout=60, status=self.status) - # Now try to create a session with rank 1 by accessing a dir known to - # be there, if buggy, this should cause the rank 1 to crash: - self.mount_b.run_shell(["ls", "a"]) + def test_ephemeral_pin_dist_override_before(self): + """ + That a conventional export pin overrides the distributed policy _before_ distributed policy is set. + """ - # Check if rank1 changed (standby tookover?) - new_rank1 = self.fs.get_rank(rank=1) - self.assertEqual(rank1['gid'], new_rank1['gid']) + count = 10 + self._setup_tree(count=count) + test = [] + for i in range(count): + path = f"tree/{i}" + self.mount_a.setfattr(path, "ceph.dir.pin", "1") + test.append(("/"+path, 1)) + self.mount_a.setfattr("tree", "ceph.dir.pin.distributed", "1") + time.sleep(10) # for something to not happen... + self._wait_subtrees(test, timeout=60, status=self.status, rank="all", path="/tree/") + + def test_ephemeral_pin_dist_override_after(self): + """ + That a conventional export pin overrides the distributed policy _after_ distributed policy is set. + """ + + self._setup_tree(count=10, distributed=True) + subtrees = self._wait_distributed_subtrees(10, status=self.status, rank="all") + victim = None + test = [] + for s in subtrees: + path = s['dir']['path'] + auth = s['auth_first'] + if auth in (0, 2) and victim is None: + victim = path + self.mount_a.setfattr(victim[1:], "ceph.dir.pin", "1") + test.append((victim, 1)) + else: + test.append((path, auth)) + self.assertIsNotNone(victim) + self._wait_subtrees(test, status=self.status, rank="all", path="/tree/") + + def test_ephemeral_pin_dist_failover(self): + """ + That MDS failover does not cause unnecessary migrations. + """ + + # pin /tree so it does not export during failover + self._setup_tree(distributed=True, export=0) + self._wait_distributed_subtrees(100, status=self.status, rank="all") + #test = [(s['dir']['path'], s['auth_first']) for s in subtrees] + before = self.fs.ranks_perf(lambda p: p['mds']['exported']) + log.info(f"export stats: {before}") + self.fs.rank_fail(rank=1) + self.status = self.fs.wait_for_daemons() + time.sleep(10) # waiting for something to not happen + after = self.fs.ranks_perf(lambda p: p['mds']['exported']) + log.info(f"export stats: {after}") + self.assertEqual(before, after) + + def test_ephemeral_pin_distribution(self): + """ + That ephemerally pinned subtrees are somewhat evenly distributed. + """ + + self.fs.set_max_mds(3) + self.status = self.fs.wait_for_daemons() + + count = 1000 + self._setup_tree(count=count, distributed=True) + subtrees = self._wait_distributed_subtrees(count, status=self.status, rank="all") + nsubtrees = len(subtrees) + + # Check if distribution is uniform + rank0 = list(filter(lambda x: x['auth_first'] == 0, subtrees)) + rank1 = list(filter(lambda x: x['auth_first'] == 1, subtrees)) + rank2 = list(filter(lambda x: x['auth_first'] == 2, subtrees)) + self.assertGreaterEqual(len(rank0)/nsubtrees, 0.2) + self.assertGreaterEqual(len(rank1)/nsubtrees, 0.2) + self.assertGreaterEqual(len(rank2)/nsubtrees, 0.2) + + def test_ephemeral_random(self): + """ + That 100% randomness causes all children to be pinned. + """ + self._setup_tree(random=1.0) + self._wait_random_subtrees(100, status=self.status, rank="all") + + def test_ephemeral_random_max(self): + """ + That the config mds_export_ephemeral_random_max is not exceeded. + """ + + r = 0.5 + count = 1000 + self._setup_tree(count=count, random=r) + subtrees = self._wait_random_subtrees(int(r*count*.75), status=self.status, rank="all") + self.config_set('mds', 'mds_export_ephemeral_random_max', 0.01) + self._setup_tree(path="tree/new", count=count) + time.sleep(30) # for something not to happen... + subtrees = self._get_subtrees(status=self.status, rank="all", path="tree/new/") + self.assertLessEqual(len(subtrees), int(.01*count*1.25)) + + def test_ephemeral_random_max_config(self): + """ + That the config mds_export_ephemeral_random_max config rejects new OOB policies. + """ + + self.config_set('mds', 'mds_export_ephemeral_random_max', 0.01) + try: + p = self._setup_tree(count=1, random=0.02, wait=False) + p.wait() + except CommandFailedError as e: + log.info(f"{e}") + self.assertIn("Invalid", p.stderr.getvalue()) + else: + raise RuntimeError("mds_export_ephemeral_random_max ignored!") + + def test_ephemeral_random_dist(self): + """ + That ephemeral random and distributed can coexist with each other. + """ + + self._setup_tree(random=1.0, distributed=True, export=0) + self._wait_distributed_subtrees(100, status=self.status) + self._wait_random_subtrees(100, status=self.status) + + def test_ephemeral_random_pin_override_before(self): + """ + That a conventional export pin overrides the random policy before creating new directories. + """ + + self._setup_tree(count=0, random=1.0) + self._setup_tree(path="tree/pin", count=10, export=1) + self._wait_subtrees([("/tree/pin", 1)], status=self.status, rank=1, path="/tree/pin") + + def test_ephemeral_random_pin_override_after(self): + """ + That a conventional export pin overrides the random policy after creating new directories. + """ + + count = 10 + self._setup_tree(count=0, random=1.0) + self._setup_tree(path="tree/pin", count=count) + self._wait_random_subtrees(count+1, status=self.status, rank="all") + self.mount_a.setfattr("tree/pin", "ceph.dir.pin", "1") + self._wait_subtrees([("/tree/pin", 1)], status=self.status, rank=1, path="/tree/pin") + + def test_ephemeral_randomness(self): + """ + That the randomness is reasonable. + """ + + r = random.uniform(0.25, 0.75) # ratios don't work for small r! + count = 1000 + self._setup_tree(count=count, random=r) + subtrees = self._wait_random_subtrees(int(r*count*.50), status=self.status, rank="all") + time.sleep(30) # for max to not be exceeded + subtrees = self._wait_random_subtrees(int(r*count*.50), status=self.status, rank="all") + self.assertLessEqual(len(subtrees), int(r*count*1.50)) + + def test_ephemeral_random_cache_drop(self): + """ + That the random ephemeral pin does not prevent empty (nothing in cache) subtree merging. + """ + + count = 100 + self._setup_tree(count=count, random=1.0) + self._wait_random_subtrees(count, status=self.status, rank="all") + self.mount_a.umount_wait() # release all caps + def _drop(): + self.fs.ranks_tell(["cache", "drop"], status=self.status) + self._wait_subtrees([], status=self.status, action=_drop) + + def test_ephemeral_random_failover(self): + """ + That the random ephemeral pins stay pinned across MDS failover. + """ + + count = 100 + r = 0.5 + self._setup_tree(count=count, random=r, export=0) + # wait for all random subtrees to be created, not a specific count + time.sleep(30) + subtrees = self._wait_random_subtrees(1, status=self.status, rank=1) + test = [(s['dir']['path'], s['auth_first']) for s in subtrees] + before = self.fs.ranks_perf(lambda p: p['mds']['exported']) + log.info(f"export stats: {before}") + self.fs.rank_fail(rank=1) + self.status = self.fs.wait_for_daemons() + time.sleep(30) # waiting for something to not happen + self._wait_subtrees(test, status=self.status, rank=1) + after = self.fs.ranks_perf(lambda p: p['mds']['exported']) + log.info(f"export stats: {after}") + self.assertEqual(before, after) + + def test_ephemeral_pin_grow_mds(self): + """ + That consistent hashing works to reduce the number of migrations. + """ + + self.fs.set_max_mds(2) + self.status = self.fs.wait_for_daemons() + + self._setup_tree(distributed=True) + subtrees_old = self._wait_distributed_subtrees(100, status=self.status, rank="all") + + self.fs.set_max_mds(3) + self.status = self.fs.wait_for_daemons() + + # Sleeping for a while to allow the ephemeral pin migrations to complete + time.sleep(30) + + subtrees_new = self._wait_distributed_subtrees(100, status=self.status, rank="all") + count = 0 + for old_subtree in subtrees_old: + for new_subtree in subtrees_new: + if (old_subtree['dir']['path'] == new_subtree['dir']['path']) and (old_subtree['auth_first'] != new_subtree['auth_first']): + count = count + 1 + break + + log.info("{0} migrations have occured due to the cluster resizing".format(count)) + # ~50% of subtrees from the two rank will migrate to another rank + self.assertLessEqual((count/len(subtrees_old)), (0.5)*1.25) # with 25% overbudget + + def test_ephemeral_pin_shrink_mds(self): + """ + That consistent hashing works to reduce the number of migrations. + """ + + self.fs.set_max_mds(3) + self.status = self.fs.wait_for_daemons() + + self._setup_tree(distributed=True) + subtrees_old = self._wait_distributed_subtrees(100, status=self.status, rank="all") + + self.fs.set_max_mds(2) + self.status = self.fs.wait_for_daemons() + time.sleep(30) + + subtrees_new = self._wait_distributed_subtrees(100, status=self.status, rank="all") + count = 0 + for old_subtree in subtrees_old: + for new_subtree in subtrees_new: + if (old_subtree['dir']['path'] == new_subtree['dir']['path']) and (old_subtree['auth_first'] != new_subtree['auth_first']): + count = count + 1 + break + + log.info("{0} migrations have occured due to the cluster resizing".format(count)) + # rebalancing from 3 -> 2 may cause half of rank 0/1 to move and all of rank 2 + self.assertLessEqual((count/len(subtrees_old)), (1.0/3.0/2.0 + 1.0/3.0/2.0 + 1.0/3.0)*1.25) # aka .66 with 25% overbudget diff --git a/ceph/qa/tasks/cephfs/test_journal_repair.py b/ceph/qa/tasks/cephfs/test_journal_repair.py index 61037b96d..b810e1a28 100644 --- a/ceph/qa/tasks/cephfs/test_journal_repair.py +++ b/ceph/qa/tasks/cephfs/test_journal_repair.py @@ -159,11 +159,8 @@ class TestJournalRepair(CephFSTestCase): # Set max_mds to 2 self.fs.set_max_mds(2) - - # See that we have two active MDSs - self.wait_until_equal(lambda: len(self.fs.get_active_names()), 2, 30, - reject_fn=lambda v: v > 2 or v < 1) - active_mds_names = self.fs.get_active_names() + status = self.fs.wait_for_daemons() + active_mds_names = self.fs.get_active_names(status=status) # Switch off any unneeded MDS daemons for unneeded_mds in set(self.mds_cluster.mds_ids) - set(active_mds_names): @@ -171,27 +168,13 @@ class TestJournalRepair(CephFSTestCase): self.mds_cluster.mds_fail(unneeded_mds) # Create a dir on each rank - self.mount_a.run_shell(["mkdir", "alpha"]) - self.mount_a.run_shell(["mkdir", "bravo"]) + self.mount_a.run_shell_payload("mkdir {alpha,bravo} && touch {alpha,bravo}/file") self.mount_a.setfattr("alpha/", "ceph.dir.pin", "0") self.mount_a.setfattr("bravo/", "ceph.dir.pin", "1") - def subtrees_assigned(): - got_subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=active_mds_names[0]) - - for s in got_subtrees: - if s['dir']['path'] == '/bravo': - if s['auth_first'] == 1: - return True - else: - # Should not happen - raise RuntimeError("/bravo is subtree but not rank 1!") - - return False - # Ensure the pinning has taken effect and the /bravo dir is now # migrated to rank 1. - self.wait_until_true(subtrees_assigned, 30) + self._wait_subtrees([('/bravo', 1), ('/alpha', 0)], rank=0, status=status) # Do some IO (this should be split across ranks according to # the rank-pinned dirs) diff --git a/ceph/qa/tasks/cephfs/test_nfs.py b/ceph/qa/tasks/cephfs/test_nfs.py new file mode 100644 index 000000000..12d13abb1 --- /dev/null +++ b/ceph/qa/tasks/cephfs/test_nfs.py @@ -0,0 +1,495 @@ +import errno +import json +import time +import logging +from io import BytesIO + +from tasks.mgr.mgr_test_case import MgrTestCase +from teuthology.exceptions import CommandFailedError + +log = logging.getLogger(__name__) + + +# TODO Add test for cluster update when ganesha can be deployed on multiple ports. +class TestNFS(MgrTestCase): + def _cmd(self, *args, stdin=''): + if stdin: + return self.mgr_cluster.mon_manager.raw_cluster_cmd(*args, stdin=stdin) + return self.mgr_cluster.mon_manager.raw_cluster_cmd(*args) + + def _nfs_cmd(self, *args): + return self._cmd("nfs", *args) + + def _orch_cmd(self, *args): + return self._cmd("orch", *args) + + def _sys_cmd(self, cmd): + cmd[0:0] = ['sudo'] + ret = self.ctx.cluster.run(args=cmd, check_status=False, stdout=BytesIO(), stderr=BytesIO()) + stdout = ret[0].stdout + if stdout: + return stdout.getvalue() + + def setUp(self): + super(TestNFS, self).setUp() + self.cluster_id = "test" + self.export_type = "cephfs" + self.pseudo_path = "/cephfs" + self.path = "/" + self.fs_name = "nfs-cephfs" + self.expected_name = "nfs.ganesha-test" + self.sample_export = { + "export_id": 1, + "path": self.path, + "cluster_id": self.cluster_id, + "pseudo": self.pseudo_path, + "access_type": "RW", + "squash": "no_root_squash", + "security_label": True, + "protocols": [ + 4 + ], + "transports": [ + "TCP" + ], + "fsal": { + "name": "CEPH", + "user_id": "test1", + "fs_name": self.fs_name, + "sec_label_xattr": '' + }, + "clients": [] + } + + def _check_port_status(self): + log.info("NETSTAT") + self._sys_cmd(['netstat', '-tnlp']) + + def _check_nfs_server_status(self): + res = self._sys_cmd(['systemctl', 'status', 'nfs-server']) + if isinstance(res, bytes) and b'Active: active' in res: + self._disable_nfs() + + def _disable_nfs(self): + log.info("Disabling NFS") + self._sys_cmd(['systemctl', 'disable', 'nfs-server', '--now']) + + def _check_nfs_status(self): + return self._orch_cmd('ps', '--daemon_type=nfs') + + def _check_auth_ls(self, export_id=1, check_in=False): + ''' + Tests export user id creation or deletion. + :param export_id: Denotes export number + :param check_in: Check specified export id + ''' + output = self._cmd('auth', 'ls') + if check_in: + self.assertIn(f'client.{self.cluster_id}{export_id}', output) + else: + self.assertNotIn(f'client-{self.cluster_id}', output) + + def _test_idempotency(self, cmd_func, cmd_args): + ''' + Test idempotency of commands. It first runs the TestNFS test method + for a command and then checks the result of command run again. TestNFS + test method has required checks to verify that command works. + :param cmd_func: TestNFS method + :param cmd_args: nfs command arguments to be run + ''' + cmd_func() + ret = self.mgr_cluster.mon_manager.raw_cluster_cmd_result(*cmd_args) + if ret != 0: + self.fail("Idempotency test failed") + + def _test_create_cluster(self): + ''' + Test single nfs cluster deployment. + ''' + # Disable any running nfs ganesha daemon + self._check_nfs_server_status() + self._nfs_cmd('cluster', 'create', self.export_type, self.cluster_id) + # Wait for few seconds as ganesha daemon take few seconds to be deployed + time.sleep(8) + orch_output = self._check_nfs_status() + expected_status = 'running' + # Check for expected status and daemon name (nfs.ganesha-) + if self.expected_name not in orch_output or expected_status not in orch_output: + self.fail("NFS Ganesha cluster could not be deployed") + + def _test_delete_cluster(self): + ''' + Test deletion of a single nfs cluster. + ''' + self._nfs_cmd('cluster', 'delete', self.cluster_id) + expected_output = "No daemons reported\n" + # Wait for few seconds as ganesha daemon takes few seconds to be deleted + wait_time = 10 + while wait_time <= 60: + time.sleep(wait_time) + orch_output = self._check_nfs_status() + if expected_output == orch_output: + return + wait_time += 10 + self.fail("NFS Ganesha cluster could not be deleted") + + def _test_list_cluster(self, empty=False): + ''' + Test listing of deployed nfs clusters. If nfs cluster is deployed then + it checks for expected cluster id. Otherwise checks nothing is listed. + :param empty: If true it denotes no cluster is deployed. + ''' + if empty: + cluster_id = '' + else: + cluster_id = self.cluster_id + nfs_output = self._nfs_cmd('cluster', 'ls') + self.assertEqual(cluster_id, nfs_output.strip()) + + def _create_export(self, export_id, create_fs=False, extra_cmd=None): + ''' + Test creation of a single export. + :param export_id: Denotes export number + :param create_fs: If false filesytem exists. Otherwise create it. + :param extra_cmd: List of extra arguments for creating export. + ''' + if create_fs: + self._cmd('fs', 'volume', 'create', self.fs_name) + export_cmd = ['nfs', 'export', 'create', 'cephfs', self.fs_name, self.cluster_id] + if isinstance(extra_cmd, list): + export_cmd.extend(extra_cmd) + else: + export_cmd.append(self.pseudo_path) + # Runs the nfs export create command + self._cmd(*export_cmd) + # Check if user id for export is created + self._check_auth_ls(export_id, check_in=True) + res = self._sys_cmd(['rados', '-p', 'nfs-ganesha', '-N', self.cluster_id, 'get', + f'export-{export_id}', '-']) + # Check if export object is created + if res == b'': + self.fail("Export cannot be created") + + def _create_default_export(self): + ''' + Deploy a single nfs cluster and create export with default options. + ''' + self._test_create_cluster() + self._create_export(export_id='1', create_fs=True) + + def _delete_export(self): + ''' + Delete an export. + ''' + self._nfs_cmd('export', 'delete', self.cluster_id, self.pseudo_path) + self._check_auth_ls() + + def _test_list_export(self): + ''' + Test listing of created exports. + ''' + nfs_output = json.loads(self._nfs_cmd('export', 'ls', self.cluster_id)) + self.assertIn(self.pseudo_path, nfs_output) + + def _test_list_detailed(self, sub_vol_path): + ''' + Test listing of created exports with detailed option. + :param sub_vol_path: Denotes path of subvolume + ''' + nfs_output = json.loads(self._nfs_cmd('export', 'ls', self.cluster_id, '--detailed')) + # Export-1 with default values (access type = rw and path = '\') + self.assertDictEqual(self.sample_export, nfs_output[0]) + # Export-2 with r only + self.sample_export['export_id'] = 2 + self.sample_export['pseudo'] = self.pseudo_path + '1' + self.sample_export['access_type'] = 'RO' + self.sample_export['fsal']['user_id'] = self.cluster_id + '2' + self.assertDictEqual(self.sample_export, nfs_output[1]) + # Export-3 for subvolume with r only + self.sample_export['export_id'] = 3 + self.sample_export['path'] = sub_vol_path + self.sample_export['pseudo'] = self.pseudo_path + '2' + self.sample_export['fsal']['user_id'] = self.cluster_id + '3' + self.assertDictEqual(self.sample_export, nfs_output[2]) + # Export-4 for subvolume + self.sample_export['export_id'] = 4 + self.sample_export['pseudo'] = self.pseudo_path + '3' + self.sample_export['access_type'] = 'RW' + self.sample_export['fsal']['user_id'] = self.cluster_id + '4' + self.assertDictEqual(self.sample_export, nfs_output[3]) + + def _test_get_export(self): + ''' + Test fetching of created export. + ''' + nfs_output = json.loads(self._nfs_cmd('export', 'get', self.cluster_id, self.pseudo_path)) + self.assertDictEqual(self.sample_export, nfs_output) + + def _check_export_obj_deleted(self, conf_obj=False): + ''' + Test if export or config object are deleted successfully. + :param conf_obj: It denotes config object needs to be checked + ''' + rados_obj_ls = self._sys_cmd(['rados', '-p', 'nfs-ganesha', '-N', self.cluster_id, 'ls']) + + if b'export-' in rados_obj_ls or (conf_obj and b'conf-nfs' in rados_obj_ls): + self.fail("Delete export failed") + + def test_create_and_delete_cluster(self): + ''' + Test successful creation and deletion of the nfs cluster. + ''' + self._test_create_cluster() + self._test_list_cluster() + self._test_delete_cluster() + # List clusters again to ensure no cluster is shown + self._test_list_cluster(empty=True) + + def test_create_delete_cluster_idempotency(self): + ''' + Test idempotency of cluster create and delete commands. + ''' + self._test_idempotency(self._test_create_cluster, ['nfs', 'cluster', 'create', self.export_type, + self.cluster_id]) + self._test_idempotency(self._test_delete_cluster, ['nfs', 'cluster', 'delete', self.cluster_id]) + + def test_create_cluster_with_invalid_cluster_id(self): + ''' + Test nfs cluster deployment failure with invalid cluster id. + ''' + try: + invalid_cluster_id = '/cluster_test' # Only [A-Za-z0-9-_.] chars are valid + self._nfs_cmd('cluster', 'create', self.export_type, invalid_cluster_id) + self.fail(f"Cluster successfully created with invalid cluster id {invalid_cluster_id}") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.EINVAL: + raise + + def test_create_cluster_with_invalid_export_type(self): + ''' + Test nfs cluster deployment failure with invalid export type. + ''' + try: + invalid_export_type = 'rgw' # Only cephfs is valid + self._nfs_cmd('cluster', 'create', invalid_export_type, self.cluster_id) + self.fail(f"Cluster successfully created with invalid export type {invalid_export_type}") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.EINVAL: + raise + + def test_create_and_delete_export(self): + ''' + Test successful creation and deletion of the cephfs export. + ''' + self._create_default_export() + self._test_get_export() + self._delete_export() + # Check if rados export object is deleted + self._check_export_obj_deleted() + self._test_delete_cluster() + + def test_create_delete_export_idempotency(self): + ''' + Test idempotency of export create and delete commands. + ''' + self._test_idempotency(self._create_default_export, ['nfs', 'export', 'create', 'cephfs', + self.fs_name, self.cluster_id, + self.pseudo_path]) + self._test_idempotency(self._delete_export, ['nfs', 'export', 'delete', self.cluster_id, + self.pseudo_path]) + self._test_delete_cluster() + + def test_create_multiple_exports(self): + ''' + Test creating multiple exports with different access type and path. + ''' + # Export-1 with default values (access type = rw and path = '\') + self._create_default_export() + # Export-2 with r only + self._create_export(export_id='2', extra_cmd=[self.pseudo_path+'1', '--readonly']) + # Export-3 for subvolume with r only + self._cmd('fs', 'subvolume', 'create', self.fs_name, 'sub_vol') + fs_path = self._cmd('fs', 'subvolume', 'getpath', self.fs_name, 'sub_vol').strip() + self._create_export(export_id='3', extra_cmd=[self.pseudo_path+'2', '--readonly', fs_path]) + # Export-4 for subvolume + self._create_export(export_id='4', extra_cmd=[self.pseudo_path+'3', fs_path]) + # Check if exports gets listed + self._test_list_detailed(fs_path) + self._test_delete_cluster() + # Check if rados ganesha conf object is deleted + self._check_export_obj_deleted(conf_obj=True) + self._check_auth_ls() + + def test_exports_on_mgr_restart(self): + ''' + Test export availability on restarting mgr. + ''' + self._create_default_export() + # unload and load module will restart the mgr + self._unload_module("cephadm") + self._load_module("cephadm") + self._orch_cmd("set", "backend", "cephadm") + # Checks if created export is listed + self._test_list_export() + self._delete_export() + self._test_delete_cluster() + + def test_export_create_with_non_existing_fsname(self): + ''' + Test creating export with non-existing filesystem. + ''' + try: + fs_name = 'nfs-test' + self._test_create_cluster() + self._nfs_cmd('export', 'create', 'cephfs', fs_name, self.cluster_id, self.pseudo_path) + self.fail(f"Export created with non-existing filesystem {fs_name}") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.ENOENT: + raise + finally: + self._test_delete_cluster() + + def test_export_create_with_non_existing_clusterid(self): + ''' + Test creating cephfs export with non-existing nfs cluster. + ''' + try: + cluster_id = 'invalidtest' + self._nfs_cmd('export', 'create', 'cephfs', self.fs_name, cluster_id, self.pseudo_path) + self.fail(f"Export created with non-existing cluster id {cluster_id}") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.ENOENT: + raise + + def test_export_create_with_relative_pseudo_path_and_root_directory(self): + ''' + Test creating cephfs export with relative or '/' pseudo path. + ''' + def check_pseudo_path(pseudo_path): + try: + self._nfs_cmd('export', 'create', 'cephfs', self.fs_name, self.cluster_id, + pseudo_path) + self.fail(f"Export created for {pseudo_path}") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.EINVAL: + raise + + self._test_create_cluster() + self._cmd('fs', 'volume', 'create', self.fs_name) + check_pseudo_path('invalidpath') + check_pseudo_path('/') + check_pseudo_path('//') + self._test_delete_cluster() + + def test_cluster_info(self): + ''' + Test cluster info outputs correct ip and hostname + ''' + self._test_create_cluster() + info_output = json.loads(self._nfs_cmd('cluster', 'info', self.cluster_id)) + host_details = {self.cluster_id: [{ + "hostname": self._sys_cmd(['hostname']).decode("utf-8").strip(), + "ip": list(set(self._sys_cmd(['hostname', '-I']).decode("utf-8").split())), + "port": 2049 + }]} + self.assertDictEqual(info_output, host_details) + self._test_delete_cluster() + + def test_cluster_set_reset_user_config(self): + ''' + Test cluster is created using user config and reverts back to default + config on reset. + ''' + self._test_create_cluster() + time.sleep(30) + + pool = 'nfs-ganesha' + user_id = 'test' + fs_name = 'user_test_fs' + self._cmd('fs', 'volume', 'create', fs_name) + time.sleep(20) + key = self._cmd('auth', 'get-or-create-key', f'client.{user_id}', 'mon', + 'allow r', 'osd', + f'allow rw pool={pool} namespace={self.cluster_id}, allow rw tag cephfs data={fs_name}', + 'mds', f'allow rw path={self.path}').strip() + config = f""" LOG {{ + Default_log_level = FULL_DEBUG; + }} + + EXPORT {{ + Export_Id = 100; + Transports = TCP; + Path = /; + Pseudo = /ceph/; + Protocols = 4; + Access_Type = RW; + Attr_Expiration_Time = 0; + Squash = None; + FSAL {{ + Name = CEPH; + Filesystem = {fs_name}; + User_Id = {user_id}; + Secret_Access_Key = '{key}'; + }} + }}""" + #{'test': [{'hostname': 'smithi068', 'ip': ['172.21.15.68'], 'port': 2049}]} + info_output = json.loads(self._nfs_cmd('cluster', 'info', self.cluster_id))['test'][0] + mnt_cmd = ['sudo', 'mount', '-t', 'nfs', '-o', f'port={info_output["port"]}', f'{info_output["ip"][0]}:/ceph', '/mnt'] + MNT_FAILED = 32 + self.ctx.cluster.run(args=['sudo', 'ceph', 'nfs', 'cluster', 'config', + 'set', self.cluster_id, '-i', '-'], stdin=config) + time.sleep(30) + log.info(self._sys_cmd(['rados', '-p', 'nfs-ganesha', '-N', self.cluster_id, 'ls'])) + res = self._sys_cmd(['rados', '-p', pool, '-N', self.cluster_id, 'get', + f'userconf-nfs.ganesha-{user_id}', '-']) + self.assertEqual(config, res.decode('utf-8')) + self.ctx.cluster.run(args=mnt_cmd) + self.ctx.cluster.run(args=['sudo', 'touch', '/mnt/test']) + out_mnt = self._sys_cmd(['sudo', 'ls', '/mnt']) + self.assertEqual(out_mnt, b'test\n') + self.ctx.cluster.run(args=['sudo', 'umount', '/mnt']) + self._nfs_cmd('cluster', 'config', 'reset', self.cluster_id) + rados_obj_ls = self._sys_cmd(['rados', '-p', 'nfs-ganesha', '-N', self.cluster_id, 'ls']) + if b'conf-nfs' not in rados_obj_ls and b'userconf-nfs' in rados_obj_ls: + self.fail("User config not deleted") + time.sleep(30) + try: + self.ctx.cluster.run(args=mnt_cmd) + except CommandFailedError as e: + if e.exitstatus != MNT_FAILED: + raise + self._cmd('fs', 'volume', 'rm', fs_name, '--yes-i-really-mean-it') + self._test_delete_cluster() + time.sleep(30) + + def test_cluster_set_user_config_with_non_existing_clusterid(self): + ''' + Test setting user config for non-existing nfs cluster. + ''' + try: + cluster_id = 'invalidtest' + self.ctx.cluster.run(args=['sudo', 'ceph', 'nfs', 'cluster', + 'config', 'set', self.cluster_id, '-i', '-'], stdin='testing') + self.fail(f"User config set for non-existing cluster {cluster_id}") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.ENOENT: + raise + + def test_cluster_reset_user_config_with_non_existing_clusterid(self): + ''' + Test resetting user config for non-existing nfs cluster. + ''' + try: + cluster_id = 'invalidtest' + self._nfs_cmd('cluster', 'config', 'reset', cluster_id) + self.fail(f"User config reset for non-existing cluster {cluster_id}") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.ENOENT: + raise diff --git a/ceph/qa/tasks/cephfs/test_sessionmap.py b/ceph/qa/tasks/cephfs/test_sessionmap.py index bdcde71d0..87e789770 100644 --- a/ceph/qa/tasks/cephfs/test_sessionmap.py +++ b/ceph/qa/tasks/cephfs/test_sessionmap.py @@ -196,13 +196,12 @@ class TestSessionMap(CephFSTestCase): self.skipTest("Requires FUSE client to use is_blacklisted()") self.fs.set_max_mds(2) - self.fs.wait_for_daemons() - status = self.fs.status() + status = self.fs.wait_for_daemons() - self.mount_a.run_shell(["mkdir", "d0", "d1"]) + self.mount_a.run_shell_payload("mkdir {d0,d1} && touch {d0,d1}/file") self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d1", "ceph.dir.pin", "1") - self._wait_subtrees(status, 0, [('/d0', 0), ('/d1', 1)]) + self._wait_subtrees([('/d0', 0), ('/d1', 1)], status=status) self.mount_a.run_shell(["touch", "d0/f0"]) self.mount_a.run_shell(["touch", "d1/f0"]) diff --git a/ceph/qa/tasks/cephfs/test_snapshots.py b/ceph/qa/tasks/cephfs/test_snapshots.py index 0a35d99d4..40a09f3a8 100644 --- a/ceph/qa/tasks/cephfs/test_snapshots.py +++ b/ceph/qa/tasks/cephfs/test_snapshots.py @@ -55,7 +55,7 @@ class TestSnapshots(CephFSTestCase): # setup subtrees self.mount_a.run_shell(["mkdir", "-p", "d1/dir"]) self.mount_a.setfattr("d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(1, '/d1', status=status), timeout=30) + self._wait_subtrees([("/d1", 1)], rank=1, path="/d1") last_created = self._get_last_created_snap(rank=0,status=status) @@ -231,9 +231,7 @@ class TestSnapshots(CephFSTestCase): self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d0/d1", "ceph.dir.pin", "1") self.mount_a.setfattr("d0/d2", "ceph.dir.pin", "2") - self.wait_until_true(lambda: self._check_subtree(2, '/d0/d2', status=status), timeout=30) - self.wait_until_true(lambda: self._check_subtree(1, '/d0/d1', status=status), timeout=5) - self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) + self._wait_subtrees([("/d0", 0), ("/d0/d1", 1), ("/d0/d2", 2)], rank="all", status=status, path="/d0") def _check_snapclient_cache(snaps_dump, cache_dump=None, rank=0): if cache_dump is None: @@ -354,11 +352,10 @@ class TestSnapshots(CephFSTestCase): self.fs.set_max_mds(2) status = self.fs.wait_for_daemons() - self.mount_a.run_shell(["mkdir", "-p", "d0/d1"]) + self.mount_a.run_shell(["mkdir", "-p", "d0/d1/empty"]) self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d0/d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(1, '/d0/d1', status=status), timeout=30) - self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) + self._wait_subtrees([("/d0", 0), ("/d0/d1", 1)], rank="all", status=status, path="/d0") self.mount_a.write_test_pattern("d0/d1/file_a", 8 * 1024 * 1024) self.mount_a.run_shell(["mkdir", "d0/.snap/s1"]) @@ -376,11 +373,10 @@ class TestSnapshots(CephFSTestCase): self.fs.set_max_mds(2) status = self.fs.wait_for_daemons() - self.mount_a.run_shell(["mkdir", "d0", "d1"]) + self.mount_a.run_shell_payload("mkdir -p {d0,d1}/empty") self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(1, '/d1', status=status), timeout=30) - self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) + self._wait_subtrees([("/d0", 0), ("/d1", 1)], rank=0, status=status) self.mount_a.run_shell(["mkdir", "d0/d3"]) self.mount_a.run_shell(["mkdir", "d0/.snap/s1"]) @@ -404,12 +400,11 @@ class TestSnapshots(CephFSTestCase): self.fs.set_max_mds(2) status = self.fs.wait_for_daemons() - self.mount_a.run_shell(["mkdir", "d0", "d1"]) + self.mount_a.run_shell_payload("mkdir -p {d0,d1}/empty") self.mount_a.setfattr("d0", "ceph.dir.pin", "0") self.mount_a.setfattr("d1", "ceph.dir.pin", "1") - self.wait_until_true(lambda: self._check_subtree(1, '/d1', status=status), timeout=30) - self.wait_until_true(lambda: self._check_subtree(0, '/d0', status=status), timeout=5) + self._wait_subtrees([("/d0", 0), ("/d1", 1)], rank=0, status=status) self.mount_a.run_python(dedent(""" import os diff --git a/ceph/qa/tasks/cephfs/test_strays.py b/ceph/qa/tasks/cephfs/test_strays.py index a5058441e..4dd70d3ee 100644 --- a/ceph/qa/tasks/cephfs/test_strays.py +++ b/ceph/qa/tasks/cephfs/test_strays.py @@ -517,34 +517,16 @@ class TestStrays(CephFSTestCase): return rank_0_id, rank_1_id - def _force_migrate(self, to_id, path, watch_ino): + def _force_migrate(self, path, rank=1): """ :param to_id: MDS id to move it to :param path: Filesystem path (string) to move :param watch_ino: Inode number to look for at destination to confirm move :return: None """ - self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "1", path]) - - # Poll the MDS cache dump to watch for the export completing - migrated = False - migrate_timeout = 60 - migrate_elapsed = 0 - while not migrated: - data = self.fs.mds_asok(["dump", "cache"], to_id) - for inode_data in data: - if inode_data['ino'] == watch_ino: - log.debug("Found ino in cache: {0}".format(json.dumps(inode_data, indent=2))) - if inode_data['is_auth'] is True: - migrated = True - break - - if not migrated: - if migrate_elapsed > migrate_timeout: - raise RuntimeError("Migration hasn't happened after {0}s!".format(migrate_elapsed)) - else: - migrate_elapsed += 1 - time.sleep(1) + self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", str(rank), path]) + rpath = "/"+path + self._wait_subtrees([(rpath, rank)], rank=rank, path=rpath) def _is_stopped(self, rank): mds_map = self.fs.get_mds_map() @@ -565,8 +547,7 @@ class TestStrays(CephFSTestCase): self.mount_a.create_n_files("delete_me/file", file_count) - self._force_migrate(rank_1_id, "delete_me", - self.mount_a.path_to_ino("delete_me/file_0")) + self._force_migrate("delete_me") self.mount_a.run_shell(["rm", "-rf", Raw("delete_me/*")]) self.mount_a.umount_wait() @@ -610,26 +591,21 @@ class TestStrays(CephFSTestCase): # Create a non-purgeable stray in a ~mds1 stray directory # by doing a hard link and deleting the original file - self.mount_a.run_shell(["mkdir", "dir_1", "dir_2"]) - self.mount_a.run_shell(["touch", "dir_1/original"]) - self.mount_a.run_shell(["ln", "dir_1/original", "dir_2/linkto"]) + self.mount_a.run_shell_payload(""" +mkdir dir_1 dir_2 +touch dir_1/original +ln dir_1/original dir_2/linkto +""") - self._force_migrate(rank_1_id, "dir_1", - self.mount_a.path_to_ino("dir_1/original")) + self._force_migrate("dir_1") + self._force_migrate("dir_2", rank=0) # empty mds cache. otherwise mds reintegrates stray when unlink finishes self.mount_a.umount_wait() - self.fs.mds_asok(['flush', 'journal'], rank_0_id) self.fs.mds_asok(['flush', 'journal'], rank_1_id) - self.fs.mds_fail_restart() - self.fs.wait_for_daemons() - - active_mds_names = self.fs.get_active_names() - rank_0_id = active_mds_names[0] - rank_1_id = active_mds_names[1] + self.fs.mds_asok(['cache', 'drop'], rank_1_id) self.mount_a.mount_wait() - self.mount_a.run_shell(["rm", "-f", "dir_1/original"]) self.mount_a.umount_wait() @@ -955,8 +931,7 @@ class TestStrays(CephFSTestCase): self.mount_a.create_n_files("delete_me/file", file_count) - self._force_migrate(rank_1_id, "delete_me", - self.mount_a.path_to_ino("delete_me/file_0")) + self._force_migrate("delete_me") begin = datetime.datetime.now() self.mount_a.run_shell(["rm", "-rf", Raw("delete_me/*")]) @@ -969,4 +944,3 @@ class TestStrays(CephFSTestCase): duration = (end - begin).total_seconds() self.assertLess(duration, (file_count * tick_period) * 0.25) - diff --git a/ceph/qa/tasks/cephfs/test_volumes.py b/ceph/qa/tasks/cephfs/test_volumes.py index 11c23605a..88fbe04cd 100644 --- a/ceph/qa/tasks/cephfs/test_volumes.py +++ b/ceph/qa/tasks/cephfs/test_volumes.py @@ -21,6 +21,7 @@ class TestVolumes(CephFSTestCase): # for filling subvolume with data CLIENTS_REQUIRED = 1 + MDSS_REQUIRED = 2 # io defaults DEFAULT_FILE_SIZE = 1 # MB @@ -29,6 +30,9 @@ class TestVolumes(CephFSTestCase): def _fs_cmd(self, *args): return self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", *args) + def _raw_cmd(self, *args): + return self.mgr_cluster.mon_manager.raw_cluster_cmd(*args) + def __check_clone_state(self, state, clone, clone_group=None, timo=120): check = 0 args = ["clone", "status", self.volname, clone] @@ -105,28 +109,33 @@ class TestVolumes(CephFSTestCase): self._verify_clone_attrs(subvolume, clone, source_group=source_group, clone_group=clone_group) def _generate_random_volume_name(self, count=1): - r = random.sample(range(10000), count) - volumes = ["{0}_{1}".format(TestVolumes.TEST_VOLUME_PREFIX, c) for c in r] + n = self.volume_start + volumes = [f"{TestVolumes.TEST_VOLUME_PREFIX}_{i:016}" for i in range(n, n+count)] + self.volume_start += count return volumes[0] if count == 1 else volumes def _generate_random_subvolume_name(self, count=1): - r = random.sample(range(10000), count) - subvolumes = ["{0}_{1}".format(TestVolumes.TEST_SUBVOLUME_PREFIX, c) for c in r] + n = self.subvolume_start + subvolumes = [f"{TestVolumes.TEST_SUBVOLUME_PREFIX}_{i:016}" for i in range(n, n+count)] + self.subvolume_start += count return subvolumes[0] if count == 1 else subvolumes def _generate_random_group_name(self, count=1): - r = random.sample(range(100), count) - groups = ["{0}_{1}".format(TestVolumes.TEST_GROUP_PREFIX, c) for c in r] + n = self.group_start + groups = [f"{TestVolumes.TEST_GROUP_PREFIX}_{i:016}" for i in range(n, n+count)] + self.group_start += count return groups[0] if count == 1 else groups def _generate_random_snapshot_name(self, count=1): - r = random.sample(range(100), count) - snaps = ["{0}_{1}".format(TestVolumes.TEST_SNAPSHOT_PREFIX, c) for c in r] + n = self.snapshot_start + snaps = [f"{TestVolumes.TEST_SNAPSHOT_PREFIX}_{i:016}" for i in range(n, n+count)] + self.snapshot_start += count return snaps[0] if count == 1 else snaps def _generate_random_clone_name(self, count=1): - r = random.sample(range(1000), count) - clones = ["{0}_{1}".format(TestVolumes.TEST_CLONE_PREFIX, c) for c in r] + n = self.clone_start + clones = [f"{TestVolumes.TEST_CLONE_PREFIX}_{i:016}" for i in range(n, n+count)] + self.clone_start += count return clones[0] if count == 1 else clones def _enable_multi_fs(self): @@ -225,6 +234,12 @@ class TestVolumes(CephFSTestCase): self.vol_created = False self._enable_multi_fs() self._create_or_reuse_test_volume() + self.config_set('mon', 'mon_allow_pool_delete', True) + self.volume_start = random.randint(1, (1<<20)) + self.subvolume_start = random.randint(1, (1<<20)) + self.group_start = random.randint(1, (1<<20)) + self.snapshot_start = random.randint(1, (1<<20)) + self.clone_start = random.randint(1, (1<<20)) def tearDown(self): if self.vol_created: @@ -310,6 +325,54 @@ class TestVolumes(CephFSTestCase): else: raise RuntimeError("expected the 'fs volume rm' command to fail.") + def test_volume_rm_arbitrary_pool_removal(self): + """ + That the arbitrary pool added to the volume out of band is removed + successfully on volume removal. + """ + new_pool = "new_pool" + # add arbitrary data pool + self.fs.add_data_pool(new_pool) + vol_status = json.loads(self._fs_cmd("status", self.volname, "--format=json-pretty")) + self._fs_cmd("volume", "rm", self.volname, "--yes-i-really-mean-it") + + #check if fs is gone + volumes = json.loads(self._fs_cmd("volume", "ls", "--format=json-pretty")) + volnames = [volume['name'] for volume in volumes] + self.assertNotIn(self.volname, volnames) + + #check if osd pools are gone + pools = json.loads(self._raw_cmd("osd", "pool", "ls", "--format=json-pretty")) + for pool in vol_status["pools"]: + self.assertNotIn(pool["name"], pools) + + def test_volume_rm_when_mon_delete_pool_false(self): + """ + That the volume can only be removed when mon_allowd_pool_delete is set + to true and verify that the pools are removed after volume deletion. + """ + self.config_set('mon', 'mon_allow_pool_delete', False) + try: + self._fs_cmd("volume", "rm", self.volname, "--yes-i-really-mean-it") + except CommandFailedError as ce: + self.assertEqual(ce.exitstatus, errno.EPERM, + "expected the 'fs volume rm' command to fail with EPERM, " + "but it failed with {0}".format(ce.exitstatus)) + vol_status = json.loads(self._fs_cmd("status", self.volname, "--format=json-pretty")) + self.config_set('mon', 'mon_allow_pool_delete', True) + self._fs_cmd("volume", "rm", self.volname, "--yes-i-really-mean-it") + + #check if fs is gone + volumes = json.loads(self._fs_cmd("volume", "ls", "--format=json-pretty")) + volnames = [volume['name'] for volume in volumes] + self.assertNotIn(self.volname, volnames, + "volume {0} exists after removal".format(self.volname)) + #check if pools are gone + pools = json.loads(self._raw_cmd("osd", "pool", "ls", "--format=json-pretty")) + for pool in vol_status["pools"]: + self.assertNotIn(pool["name"], pools, + "pool {0} exists after volume removal".format(pool["name"])) + ### basic subvolume operations def test_subvolume_create_and_rm(self): @@ -592,6 +655,44 @@ class TestVolumes(CephFSTestCase): # verify trash dir is clean self._wait_for_trash_empty() + def test_subvolume_pin_export(self): + self.fs.set_max_mds(2) + status = self.fs.wait_for_daemons() + + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "pin", self.volname, subvolume, "export", "1") + path = self._fs_cmd("subvolume", "getpath", self.volname, subvolume) + path = os.path.dirname(path) # get subvolume path + + self._get_subtrees(status=status, rank=1) + self._wait_subtrees([(path, 1)], status=status) + + def test_subvolumegroup_pin_distributed(self): + self.fs.set_max_mds(2) + status = self.fs.wait_for_daemons() + self.config_set('mds', 'mds_export_ephemeral_distributed', True) + + group = "pinme" + self._fs_cmd("subvolumegroup", "create", self.volname, group) + self._fs_cmd("subvolumegroup", "pin", self.volname, group, "distributed", "True") + # (no effect on distribution) pin the group directory to 0 so rank 0 has all subtree bounds visible + self._fs_cmd("subvolumegroup", "pin", self.volname, group, "export", "0") + subvolumes = self._generate_random_subvolume_name(10) + for subvolume in subvolumes: + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) + self._wait_distributed_subtrees(10, status=status) + + def test_subvolume_pin_random(self): + self.fs.set_max_mds(2) + self.fs.wait_for_daemons() + self.config_set('mds', 'mds_export_ephemeral_random', True) + + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "pin", self.volname, subvolume, "random", ".01") + # no verification + def test_subvolume_create_isolated_namespace(self): """ Create subvolume in separate rados namespace @@ -792,7 +893,7 @@ class TestVolumes(CephFSTestCase): subvol_md = ["atime", "bytes_pcent", "bytes_quota", "bytes_used", "created_at", "ctime", "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "pool_namespace", - "type", "uid"] + "type", "uid", "features"] # create subvolume subvolume = self._generate_random_subvolume_name() @@ -800,37 +901,34 @@ class TestVolumes(CephFSTestCase): # get subvolume metadata subvol_info = json.loads(self._get_subvolume_info(self.volname, subvolume)) - if len(subvol_info) == 0: - raise RuntimeError("Expected the 'fs subvolume info' command to list metadata of subvolume") + self.assertNotEqual(len(subvol_info), 0, "expected the 'fs subvolume info' command to list metadata of subvolume") for md in subvol_md: - if md not in subvol_info.keys(): - raise RuntimeError("%s not present in the metadata of subvolume" % md) + self.assertIn(md, subvol_info.keys(), "'{0}' key not present in metadata of subvolume".format(md)) - if subvol_info["bytes_pcent"] != "undefined": - raise RuntimeError("bytes_pcent should be set to undefined if quota is not set") + self.assertEqual(subvol_info["bytes_pcent"], "undefined", "bytes_pcent should be set to undefined if quota is not set") + self.assertEqual(subvol_info["bytes_quota"], "infinite", "bytes_quota should be set to infinite if quota is not set") + self.assertEqual(subvol_info["pool_namespace"], "", "expected pool namespace to be empty") - if subvol_info["bytes_quota"] != "infinite": - raise RuntimeError("bytes_quota should be set to infinite if quota is not set") - self.assertEqual(subvol_info["pool_namespace"], "") + self.assertEqual(len(subvol_info["features"]), 2, + msg="expected 2 features, found '{0}' ({1})".format(len(subvol_info["features"]), subvol_info["features"])) + for feature in ['snapshot-clone', 'snapshot-autoprotect']: + self.assertIn(feature, subvol_info["features"], msg="expected feature '{0}' in subvolume".format(feature)) nsize = self.DEFAULT_FILE_SIZE*1024*1024 - try: - self._fs_cmd("subvolume", "resize", self.volname, subvolume, str(nsize)) - except CommandFailedError: - raise RuntimeError("expected the 'fs subvolume resize' command to succeed") + self._fs_cmd("subvolume", "resize", self.volname, subvolume, str(nsize)) # get subvolume metadata after quota set subvol_info = json.loads(self._get_subvolume_info(self.volname, subvolume)) - if len(subvol_info) == 0: - raise RuntimeError("Expected the 'fs subvolume info' command to list metadata of subvolume") - if subvol_info["bytes_pcent"] == "undefined": - raise RuntimeError("bytes_pcent should not be set to undefined if quota is set") + self.assertNotEqual(len(subvol_info), 0, "expected the 'fs subvolume info' command to list metadata of subvolume") - if subvol_info["bytes_quota"] == "infinite": - raise RuntimeError("bytes_quota should not be set to infinite if quota is set") + self.assertNotEqual(subvol_info["bytes_pcent"], "undefined", "bytes_pcent should not be set to undefined if quota is not set") + self.assertNotEqual(subvol_info["bytes_quota"], "infinite", "bytes_quota should not be set to infinite if quota is not set") + self.assertEqual(subvol_info["type"], "subvolume", "type should be set to subvolume") - if subvol_info["type"] != "subvolume": - raise RuntimeError("type should be set to subvolume") + self.assertEqual(len(subvol_info["features"]), 2, + msg="expected 2 features, found '{0}' ({1})".format(len(subvol_info["features"]), subvol_info["features"])) + for feature in ['snapshot-clone', 'snapshot-autoprotect']: + self.assertIn(feature, subvol_info["features"], msg="expected feature '{0}' in subvolume".format(feature)) # remove subvolumes self._fs_cmd("subvolume", "rm", self.volname, subvolume) @@ -858,18 +956,12 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -912,8 +1004,7 @@ class TestVolumes(CephFSTestCase): self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_subvolume_group_create_with_desired_data_pool_layout(self): - group1 = self._generate_random_group_name() - group2 = self._generate_random_group_name() + group1, group2 = self._generate_random_group_name(2) # create group self._fs_cmd("subvolumegroup", "create", self.volname, group1) @@ -974,8 +1065,7 @@ class TestVolumes(CephFSTestCase): raise RuntimeError("expected the 'fs subvolumegroup getpath' command to fail") def test_subvolume_create_with_desired_data_pool_layout_in_group(self): - subvol1 = self._generate_random_subvolume_name() - subvol2 = self._generate_random_subvolume_name() + subvol1, subvol2 = self._generate_random_subvolume_name(2) group = self._generate_random_group_name() # create group. this also helps set default pool layout for subvolumes @@ -1006,8 +1096,7 @@ class TestVolumes(CephFSTestCase): self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_subvolume_group_create_with_desired_mode(self): - group1 = self._generate_random_group_name() - group2 = self._generate_random_group_name() + group1, group2 = self._generate_random_group_name(2) # default mode expected_mode1 = "755" # desired mode @@ -1055,9 +1144,8 @@ class TestVolumes(CephFSTestCase): self._fs_cmd("subvolumegroup", "rm", self.volname, subvolgroupname) def test_subvolume_create_with_desired_mode_in_group(self): - subvol1 = self._generate_random_subvolume_name() - subvol2 = self._generate_random_subvolume_name() - subvol3 = self._generate_random_subvolume_name() + subvol1, subvol2, subvol3 = self._generate_random_subvolume_name(3) + group = self._generate_random_group_name() # default mode expected_mode1 = "755" @@ -1198,7 +1286,7 @@ class TestVolumes(CephFSTestCase): tests the 'fs subvolume snapshot info' command """ - snap_metadata = ["created_at", "data_pool", "has_pending_clones", "protected", "size"] + snap_metadata = ["created_at", "data_pool", "has_pending_clones", "size"] subvolume = self._generate_random_subvolume_name() snapshot = self._generate_random_snapshot_name() @@ -1212,20 +1300,13 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - snap_info = json.loads(self._get_subvolume_snapshot_info(self.volname, subvolume, snapshot)) self.assertNotEqual(len(snap_info), 0) for md in snap_metadata: if md not in snap_info: raise RuntimeError("%s not present in the metadata of subvolume snapshot" % md) - self.assertEqual(snap_info["protected"], "yes") self.assertEqual(snap_info["has_pending_clones"], "no") - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -1571,21 +1652,20 @@ class TestVolumes(CephFSTestCase): # verify trash dir is clean self._wait_for_trash_empty() - def test_subvolume_snapshot_protect_unprotect(self): + def test_subvolume_snapshot_protect_unprotect_sanity(self): + """ + Snapshot protect/unprotect commands are deprecated. This test exists to ensure that + invoking the command does not cause errors, till they are removed from a subsequent release. + """ subvolume = self._generate_random_subvolume_name() snapshot = self._generate_random_snapshot_name() + clone = self._generate_random_clone_name() # create subvolume self._fs_cmd("subvolume", "create", self.volname, subvolume) - # protect a nonexistent snapshot - try: - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - except CommandFailedError as ce: - if ce.exitstatus != errno.ENOENT: - raise RuntimeError("invalid error code when protecting a non-existing snapshot") - else: - raise RuntimeError("expected protection of non existent snapshot to fail") + # do some IO + self._do_subvolume_io(subvolume, number_of_files=64) # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) @@ -1593,23 +1673,11 @@ class TestVolumes(CephFSTestCase): # now, protect snapshot self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # protecting snapshot again, should return EEXIST - try: - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - except CommandFailedError as ce: - if ce.exitstatus != errno.EEXIST: - raise RuntimeError("invalid error code when protecting a protected snapshot") - else: - raise RuntimeError("expected protection of already protected snapshot to fail") + # schedule a clone + self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) - # remove snapshot should fail since the snapshot is protected - try: - self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) - except CommandFailedError as ce: - if ce.exitstatus != errno.EINVAL: - raise RuntimeError("invalid error code when removing a protected snapshot") - else: - raise RuntimeError("expected removal of protected snapshot to fail") + # check clone status + self._wait_for_clone_to_complete(clone) # now, unprotect snapshot self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) @@ -1617,37 +1685,12 @@ class TestVolumes(CephFSTestCase): # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) - # remove subvolume - self._fs_cmd("subvolume", "rm", self.volname, subvolume) - - # verify trash dir is clean - self._wait_for_trash_empty() - - def test_subvolume_snapshot_clone_unprotected_snapshot(self): - subvolume = self._generate_random_subvolume_name() - snapshot = self._generate_random_snapshot_name() - clone = self._generate_random_clone_name() - - # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) - - # snapshot subvolume - self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - - # clone a non protected snapshot - try: - self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) - except CommandFailedError as ce: - if ce.exitstatus != errno.EINVAL: - raise RuntimeError("invalid error code when cloning a non protected snapshot") - else: - raise RuntimeError("expected cloning of unprotected snapshot to fail") - - # remove snapshot - self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) + # verify clone + self._verify_clone(subvolume, clone) # remove subvolumes self._fs_cmd("subvolume", "rm", self.volname, subvolume) + self._fs_cmd("subvolume", "rm", self.volname, clone) # verify trash dir is clean self._wait_for_trash_empty() @@ -1666,27 +1709,12 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) - # unprotecting when a clone is in progress should fail - try: - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - except CommandFailedError as ce: - if ce.exitstatus != errno.EEXIST: - raise RuntimeError("invalid error code when unprotecting snapshot during clone") - else: - raise RuntimeError("expected unprotecting a snapshot to fail since it has pending clones") - # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -1718,18 +1746,12 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone, "--pool_layout", new_pool) # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -1765,18 +1787,12 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -1804,18 +1820,12 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone1) # check clone status self._wait_for_clone_to_complete(clone1) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -1829,18 +1839,12 @@ class TestVolumes(CephFSTestCase): # snapshot clone -- use same snap name self._fs_cmd("subvolume", "snapshot", "create", self.volname, clone1, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, clone1, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, clone1, snapshot, clone2) # check clone status self._wait_for_clone_to_complete(clone2) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, clone1, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, clone1, snapshot) @@ -1870,9 +1874,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) @@ -1882,9 +1883,6 @@ class TestVolumes(CephFSTestCase): # check clone status self._wait_for_clone_to_complete(clone, clone_group=group) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -1919,18 +1917,12 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot, group) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot, group) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone, '--group_name', group) # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot, group) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot, group) @@ -1966,9 +1958,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot, s_group) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot, s_group) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone, '--group_name', s_group, '--target_group_name', c_group) @@ -1976,9 +1965,6 @@ class TestVolumes(CephFSTestCase): # check clone status self._wait_for_clone_to_complete(clone, clone_group=c_group) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot, s_group) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot, s_group) @@ -2011,23 +1997,25 @@ class TestVolumes(CephFSTestCase): self.mount_a.run_shell(['mkdir', '-p', createpath]) # do some IO - self._do_subvolume_io(subvolume, number_of_files=32) + self._do_subvolume_io(subvolume, number_of_files=64) # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) + # snapshot should not be deletable now + try: + self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) + except CommandFailedError as ce: + self.assertEqual(ce.exitstatus, errno.EAGAIN, msg="invalid error code when removing source snapshot of a clone") + else: + self.fail("expected removing source snapshot of a clone to fail") + # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -2055,9 +2043,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -2066,7 +2051,7 @@ class TestVolumes(CephFSTestCase): self._get_subvolume_path(self.volname, clone) except CommandFailedError as ce: if ce.exitstatus != errno.EAGAIN: - raise RuntimeError("invalid error code when cloning a non protected snapshot") + raise RuntimeError("invalid error code when fetching path of an pending clone") else: raise RuntimeError("expected fetching path of an pending clone to fail") @@ -2077,8 +2062,50 @@ class TestVolumes(CephFSTestCase): subvolpath = self._get_subvolume_path(self.volname, clone) self.assertNotEqual(subvolpath, None) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) + # remove snapshot + self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) + + # verify clone + self._verify_clone(subvolume, clone) + + # remove subvolumes + self._fs_cmd("subvolume", "rm", self.volname, subvolume) + self._fs_cmd("subvolume", "rm", self.volname, clone) + + # verify trash dir is clean + self._wait_for_trash_empty() + + def test_subvolume_clone_in_progress_snapshot_rm(self): + subvolume = self._generate_random_subvolume_name() + snapshot = self._generate_random_snapshot_name() + clone = self._generate_random_clone_name() + + # create subvolume + self._fs_cmd("subvolume", "create", self.volname, subvolume) + + # do some IO + self._do_subvolume_io(subvolume, number_of_files=64) + + # snapshot subvolume + self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + + # schedule a clone + self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) + + # snapshot should not be deletable now + try: + self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) + except CommandFailedError as ce: + self.assertEqual(ce.exitstatus, errno.EAGAIN, msg="invalid error code when removing source snapshot of a clone") + else: + self.fail("expected removing source snapshot of a clone to fail") + + # check clone status + self._wait_for_clone_to_complete(clone) + + # clone should be accessible now + subvolpath = self._get_subvolume_path(self.volname, clone) + self.assertNotEqual(subvolpath, None) # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -2107,9 +2134,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -2128,9 +2152,6 @@ class TestVolumes(CephFSTestCase): subvolpath = self._get_subvolume_path(self.volname, clone) self.assertNotEqual(subvolpath, None) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -2179,9 +2200,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume1, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume1, snapshot) - # schedule a clone with target as subvolume2 try: self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume1, snapshot, subvolume2) @@ -2205,9 +2223,6 @@ class TestVolumes(CephFSTestCase): # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume1, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume1, snapshot) @@ -2240,9 +2255,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # add data pool new_pool = "new_pool" self.fs.add_data_pool(new_pool) @@ -2268,9 +2280,6 @@ class TestVolumes(CephFSTestCase): # check clone status self._wait_for_clone_to_fail(clone2) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -2305,18 +2314,12 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) # check clone status self._wait_for_clone_to_complete(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -2344,9 +2347,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -2356,9 +2356,6 @@ class TestVolumes(CephFSTestCase): # verify canceled state self._check_clone_canceled(clone) - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) @@ -2398,9 +2395,6 @@ class TestVolumes(CephFSTestCase): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) - # now, protect snapshot - self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) - # schedule clones for clone in clones: self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -2426,9 +2420,6 @@ class TestVolumes(CephFSTestCase): if ce.exitstatus != errno.EINVAL: raise RuntimeError("invalid error code when cancelling on-going clone") - # now, unprotect snapshot - self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) - # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) diff --git a/ceph/qa/tasks/mgr/dashboard/helper.py b/ceph/qa/tasks/mgr/dashboard/helper.py index 1acca1537..1a7a6951c 100644 --- a/ceph/qa/tasks/mgr/dashboard/helper.py +++ b/ceph/qa/tasks/mgr/dashboard/helper.py @@ -18,6 +18,12 @@ log = logging.getLogger(__name__) class DashboardTestCase(MgrTestCase): + # Display full error diffs + maxDiff = None + + # Increased x3 (20 -> 60) + TIMEOUT_HEALTH_CLEAR = 60 + MGRS_REQUIRED = 2 MDSS_REQUIRED = 1 REQUIRE_FILESYSTEM = True @@ -183,7 +189,7 @@ class DashboardTestCase(MgrTestCase): super(DashboardTestCase, self).setUp() if not self._loggedin and self.AUTO_AUTHENTICATE: self.login('admin', 'admin') - self.wait_for_health_clear(20) + self.wait_for_health_clear(self.TIMEOUT_HEALTH_CLEAR) @classmethod def tearDownClass(cls): @@ -467,6 +473,7 @@ JList = namedtuple('JList', ['elem_typ']) JTuple = namedtuple('JList', ['elem_typs']) +JUnion = namedtuple('JUnion', ['elem_typs']) class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])): def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None): @@ -496,6 +503,10 @@ def _validate_json(val, schema, path=[]): ... ds = JObj({'a': int, 'b': str, 'c': JList(int)}) ... _validate_json(d, ds) True + >>> _validate_json({'num': 1}, JObj({'num': JUnion([int,float])})) + True + >>> _validate_json({'num': 'a'}, JObj({'num': JUnion([int,float])})) + False """ if isinstance(schema, JAny): if not schema.none and val is None: @@ -514,6 +525,14 @@ def _validate_json(val, schema, path=[]): if isinstance(schema, JTuple): return all(_validate_json(val[i], typ, path + [i]) for i, typ in enumerate(schema.elem_typs)) + if isinstance(schema, JUnion): + for typ in schema.elem_typs: + try: + if _validate_json(val, typ, path): + return True + except _ValError: + pass + return False if isinstance(schema, JObj): if val is None and schema.none: return True diff --git a/ceph/qa/tasks/mgr/dashboard/test_auth.py b/ceph/qa/tasks/mgr/dashboard/test_auth.py index 468fe3796..e76708a9c 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_auth.py +++ b/ceph/qa/tasks/mgr/dashboard/test_auth.py @@ -6,7 +6,7 @@ import time import jwt -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JLeaf +from .helper import DashboardTestCase, JObj, JLeaf class AuthTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_cephfs.py b/ceph/qa/tasks/mgr/dashboard/test_cephfs.py index 291d4d85c..5ee39457a 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_cephfs.py +++ b/ceph/qa/tasks/mgr/dashboard/test_cephfs.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import six from contextlib import contextmanager -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList, JLeaf +from .helper import DashboardTestCase, JObj, JList, JLeaf class CephfsTest(DashboardTestCase): @@ -100,6 +100,16 @@ class CephfsTest(DashboardTestCase): self._delete("/api/cephfs/{}/client/1234".format(fs_id)) self.assertStatus(404) + def test_cephfs_evict_invalid_client_id(self): + fs_id = self.get_fs_id() + self._delete("/api/cephfs/{}/client/xyz".format(fs_id)) + self.assertStatus(400) + self.assertJsonBody({ + "component": 'cephfs', + "code": "invalid_cephfs_client_id", + "detail": "Invalid cephfs client ID xyz" + }) + def test_cephfs_get(self): fs_id = self.get_fs_id() data = self._get("/api/cephfs/{}/".format(fs_id)) @@ -135,6 +145,15 @@ class CephfsTest(DashboardTestCase): self.assertToHave(cephfs, 'id') self.assertToHave(cephfs, 'mdsmap') + def test_cephfs_get_quotas(self): + fs_id = self.get_fs_id() + data = self._get("/api/cephfs/{}/get_quotas?path=/".format(fs_id)) + self.assertStatus(200) + self.assertSchema(data, JObj({ + 'max_bytes': int, + 'max_files': int + })) + def test_cephfs_tabs(self): fs_id = self.get_fs_id() data = self._get("/ui-api/cephfs/{}/tabs".format(fs_id)) diff --git a/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py b/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py index 9f134cd87..61d18000a 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py +++ b/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import time -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase class ClusterConfigurationTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py b/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py index 33949925b..a0bca63ff 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py +++ b/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import six -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList +from .helper import DashboardTestCase, JObj, JList class CrushRuleTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py b/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py index c6c829577..12e061777 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py +++ b/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import six -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList +from .helper import DashboardTestCase, JObj, JList class ECPTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_ganesha.py b/ceph/qa/tasks/mgr/dashboard/test_ganesha.py index 0311dadda..c99e3651f 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_ganesha.py +++ b/ceph/qa/tasks/mgr/dashboard/test_ganesha.py @@ -4,7 +4,7 @@ from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase, JList, JObj class GaneshaTest(DashboardTestCase): @@ -166,3 +166,38 @@ class GaneshaTest(DashboardTestCase): self.assertIn('available', data) self.assertIn('message', data) self.assertTrue(data['available']) + + def test_ganesha_fsals(self): + data = self._get('/ui-api/nfs-ganesha/fsals') + self.assertStatus(200) + self.assertIn('CEPH', data) + + def test_ganesha_filesystems(self): + data = self._get('/ui-api/nfs-ganesha/cephfs/filesystems') + self.assertStatus(200) + self.assertSchema(data, JList(JObj({ + 'id': int, + 'name': str + }))) + + def test_ganesha_lsdir(self): + self._get('/ui-api/nfs-ganesha/lsdir') + self.assertStatus(500) + + def test_ganesha_buckets(self): + data = self._get('/ui-api/nfs-ganesha/rgw/buckets') + self.assertStatus(200) + schema = JList(str) + self.assertSchema(data, schema) + + def test_ganesha_clusters(self): + data = self._get('/ui-api/nfs-ganesha/clusters') + self.assertStatus(200) + schema = JList(str) + self.assertSchema(data, schema) + + def test_ganesha_cephx_clients(self): + data = self._get('/ui-api/nfs-ganesha/cephx/clients') + self.assertStatus(200) + schema = JList(str) + self.assertSchema(data, schema) diff --git a/ceph/qa/tasks/mgr/dashboard/test_health.py b/ceph/qa/tasks/mgr/dashboard/test_health.py index e7bfb4fab..e40918744 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_health.py +++ b/ceph/qa/tasks/mgr/dashboard/test_health.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase, JAny, JLeaf, JList, JObj +from .helper import DashboardTestCase, JAny, JLeaf, JList, JObj class HealthTest(DashboardTestCase): @@ -169,7 +169,8 @@ class HealthTest(DashboardTestCase): 'wr_bytes': int, 'compress_bytes_used': int, 'compress_under_bytes': int, - 'stored_raw': int + 'stored_raw': int, + 'avail_raw': int }), 'name': str, 'id': int diff --git a/ceph/qa/tasks/mgr/dashboard/test_host.py b/ceph/qa/tasks/mgr/dashboard/test_host.py index 407207133..da8f8849b 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_host.py +++ b/ceph/qa/tasks/mgr/dashboard/test_host.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import json -from tasks.mgr.dashboard.helper import DashboardTestCase, JList, JObj -from tasks.mgr.dashboard.test_orchestrator import test_data +from .helper import DashboardTestCase, JList, JObj +from .test_orchestrator import test_data class HostControllerTest(DashboardTestCase): @@ -90,6 +90,25 @@ class HostControllerTest(DashboardTestCase): })) }))) + def test_host_daemons(self): + hosts = self._get('{}'.format(self.URL_HOST)) + hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] + assert hosts[0] + data = self._get('{}/daemons'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) + self.assertStatus(200) + self.assertSchema(data, JList(JObj({ + 'hostname': str, + 'daemon_id': str, + 'daemon_type': str + }))) + + def test_host_smart(self): + hosts = self._get('{}'.format(self.URL_HOST)) + hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] + assert hosts[0] + self._get('{}/smart'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) + self.assertStatus(200) + class HostControllerNoOrchestratorTest(DashboardTestCase): def test_host_create(self): diff --git a/ceph/qa/tasks/mgr/dashboard/test_logs.py b/ceph/qa/tasks/mgr/dashboard/test_logs.py index 5108161ad..17d5d830c 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_logs.py +++ b/ceph/qa/tasks/mgr/dashboard/test_logs.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase, JList, JObj +from .helper import DashboardTestCase, JList, JObj class LogsTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py b/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py index ec6dbb47f..266ab2d34 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py +++ b/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import logging import requests -from tasks.mgr.dashboard.helper import DashboardTestCase, JAny, JObj, JList, JLeaf +from .helper import DashboardTestCase, JAny, JObj, JList, JLeaf logger = logging.getLogger(__name__) @@ -31,6 +31,22 @@ class MgrModuleTestCase(DashboardTestCase): class MgrModuleTest(MgrModuleTestCase): + + __options_schema = JObj({ + 'name': str, + 'type': str, + 'level': str, + 'flags': int, + 'default_value': JAny(none=True), + 'min': JAny(none=False), + 'max': JAny(none=False), + 'enum_allowed': JList(str), + 'desc': str, + 'long_desc': str, + 'tags': JList(str), + 'see_also': JList(str) + }) + def test_list_disabled_module(self): self._ceph_cmd(['mgr', 'module', 'disable', 'iostat']) self.wait_until_rest_api_accessible() @@ -51,7 +67,7 @@ class MgrModuleTest(MgrModuleTestCase): 'type': str, 'level': str, 'flags': int, - 'default_value': JAny(none=False), + 'default_value': JAny(none=True), 'min': JAny(none=False), 'max': JAny(none=False), 'enum_allowed': JList(str), @@ -85,7 +101,7 @@ class MgrModuleTest(MgrModuleTestCase): 'type': str, 'level': str, 'flags': int, - 'default_value': JAny(none=False), + 'default_value': JAny(none=True), 'min': JAny(none=False), 'max': JAny(none=False), 'enum_allowed': JList(str), @@ -99,8 +115,6 @@ class MgrModuleTest(MgrModuleTestCase): self.assertIsNotNone(module_info) self.assertTrue(module_info['enabled']) - -class MgrModuleTelemetryTest(MgrModuleTestCase): def test_get(self): data = self._get('/api/mgr/module/telemetry') self.assertStatus(200) @@ -124,37 +138,58 @@ class MgrModuleTelemetryTest(MgrModuleTestCase): 'url': str })) + def test_module_options(self): + data = self._get('/api/mgr/module/telemetry/options') + self.assertStatus(200) + schema = JObj({ + 'channel_basic': self.__options_schema, + 'channel_crash': self.__options_schema, + 'channel_device': self.__options_schema, + 'channel_ident': self.__options_schema, + 'contact': self.__options_schema, + 'description': self.__options_schema, + 'device_url': self.__options_schema, + 'enabled': self.__options_schema, + 'interval': self.__options_schema, + 'last_opt_revision': self.__options_schema, + 'leaderboard': self.__options_schema, + 'log_level': self.__options_schema, + 'log_to_cluster': self.__options_schema, + 'log_to_cluster_level': self.__options_schema, + 'log_to_file': self.__options_schema, + 'organization': self.__options_schema, + 'proxy': self.__options_schema, + 'url': self.__options_schema + }) + self.assertSchema(data, schema) + + def test_module_enable(self): + self._post('/api/mgr/module/telemetry/enable') + self.assertStatus(200) + + def test_disable(self): + self._post('/api/mgr/module/iostat/disable') + self.assertStatus(200) + def test_put(self): - self.set_config_key('config/mgr/mgr/telemetry/contact', '') - self.set_config_key('config/mgr/mgr/telemetry/description', '') - self.set_config_key('config/mgr/mgr/telemetry/enabled', 'True') - self.set_config_key('config/mgr/mgr/telemetry/interval', '72') - self.set_config_key('config/mgr/mgr/telemetry/leaderboard', 'False') - self.set_config_key('config/mgr/mgr/telemetry/organization', '') - self.set_config_key('config/mgr/mgr/telemetry/proxy', '') - self.set_config_key('config/mgr/mgr/telemetry/url', '') + self.set_config_key('config/mgr/mgr/iostat/log_level', 'critical') + self.set_config_key('config/mgr/mgr/iostat/log_to_cluster', 'False') + self.set_config_key('config/mgr/mgr/iostat/log_to_cluster_level', 'info') + self.set_config_key('config/mgr/mgr/iostat/log_to_file', 'True') self._put( - '/api/mgr/module/telemetry', + '/api/mgr/module/iostat', data={ 'config': { - 'contact': 'tux@suse.com', - 'description': 'test', - 'enabled': False, - 'interval': 4711, - 'leaderboard': True, - 'organization': 'SUSE Linux', - 'proxy': 'foo', - 'url': 'https://foo.bar/report' + 'log_level': 'debug', + 'log_to_cluster': True, + 'log_to_cluster_level': 'warning', + 'log_to_file': False } }) self.assertStatus(200) - data = self._get('/api/mgr/module/telemetry') + data = self._get('/api/mgr/module/iostat') self.assertStatus(200) - self.assertEqual(data['contact'], 'tux@suse.com') - self.assertEqual(data['description'], 'test') - self.assertFalse(data['enabled']) - self.assertEqual(data['interval'], 4711) - self.assertTrue(data['leaderboard']) - self.assertEqual(data['organization'], 'SUSE Linux') - self.assertEqual(data['proxy'], 'foo') - self.assertEqual(data['url'], 'https://foo.bar/report') + self.assertEqual(data['log_level'], 'debug') + self.assertTrue(data['log_to_cluster']) + self.assertEqual(data['log_to_cluster_level'], 'warning') + self.assertFalse(data['log_to_file']) diff --git a/ceph/qa/tasks/mgr/dashboard/test_monitor.py b/ceph/qa/tasks/mgr/dashboard/test_monitor.py index 1558cdc82..0cf7e25a2 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_monitor.py +++ b/ceph/qa/tasks/mgr/dashboard/test_monitor.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase class MonitorTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py b/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py index 4f248a1c3..9f4204379 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py +++ b/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import json -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase test_data = { diff --git a/ceph/qa/tasks/mgr/dashboard/test_osd.py b/ceph/qa/tasks/mgr/dashboard/test_osd.py index 4f2028d22..0bd3f93f3 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_osd.py +++ b/ceph/qa/tasks/mgr/dashboard/test_osd.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import json -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JAny, JList, JLeaf, JTuple +from .helper import DashboardTestCase, JObj, JAny, JList, JLeaf, JTuple class OsdTest(DashboardTestCase): @@ -61,6 +61,19 @@ class OsdTest(DashboardTestCase): self._post('/api/osd/0/scrub?deep=True') self.assertStatus(200) + def test_safe_to_delete(self): + data = self._get('/api/osd/safe_to_delete?svc_ids=0') + self.assertStatus(200) + self.assertSchema(data, JObj({ + 'is_safe_to_delete': JAny(none=True), + 'message': str + })) + self.assertTrue(data['is_safe_to_delete']) + + def test_osd_smart(self): + self._get('/api/osd/0/smart') + self.assertStatus(200) + def test_mark_out_and_in(self): self._post('/api/osd/0/mark_out') self.assertStatus(200) @@ -98,6 +111,18 @@ class OsdTest(DashboardTestCase): 'tracking_id': 'bare-5' }) self.assertStatus(201) + + # invalid method + self._task_post('/api/osd', { + 'method': 'xyz', + 'data': { + 'uuid': 'f860ca2e-757d-48ce-b74a-87052cad563f', + 'svc_id': 5 + }, + 'tracking_id': 'bare-5' + }) + self.assertStatus(400) + # Lost self._post('/api/osd/5/mark_lost') self.assertStatus(200) diff --git a/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py b/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py index 99133a77c..c01368bce 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py +++ b/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj +from .helper import DashboardTestCase, JObj class PerfCountersControllerTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_pool.py b/ceph/qa/tasks/mgr/dashboard/test_pool.py index bf40ac206..a3aac7631 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_pool.py +++ b/ceph/qa/tasks/mgr/dashboard/test_pool.py @@ -6,7 +6,7 @@ import six import time from contextlib import contextmanager -from tasks.mgr.dashboard.helper import DashboardTestCase, JAny, JList, JObj +from .helper import DashboardTestCase, JAny, JList, JObj, JUnion log = logging.getLogger(__name__) @@ -23,14 +23,16 @@ class PoolTest(DashboardTestCase): }, allow_unknown=True) pool_list_stat_schema = JObj(sub_elems={ - 'latest': int, + 'latest': JUnion([int,float]), 'rate': float, 'rates': JList(JAny(none=False)), }) pool_list_stats_schema = JObj(sub_elems={ + 'avail_raw': pool_list_stat_schema, 'bytes_used': pool_list_stat_schema, 'max_avail': pool_list_stat_schema, + 'percent_used': pool_list_stat_schema, 'rd_bytes': pool_list_stat_schema, 'wr_bytes': pool_list_stat_schema, 'rd': pool_list_stat_schema, @@ -82,7 +84,7 @@ class PoolTest(DashboardTestCase): self._task_delete('/api/pool/' + name) self.assertStatus(204) - def _validate_pool_properties(self, data, pool): + def _validate_pool_properties(self, data, pool, timeout=DashboardTestCase.TIMEOUT_HEALTH_CLEAR): for prop, value in data.items(): if prop == 'pool_type': self.assertEqual(pool['type'], value) @@ -110,13 +112,15 @@ class PoolTest(DashboardTestCase): # 1. The default value cannot be given to this method, which becomes relevant # when resetting a value, because it's not always zero. # 2. The expected `source` cannot be given to this method, and it cannot - # relibably be determined (see 1) + # reliably be determined (see 1) pass else: self.assertEqual(pool[prop], value, '{}: {} != {}'.format(prop, pool[prop], value)) - health = self._get('/api/health/minimal')['health'] - self.assertEqual(health['status'], 'HEALTH_OK', msg='health={}'.format(health)) + self.wait_until_equal(self._get_health_status, 'HEALTH_OK', timeout) + + def _get_health_status(self): + return self._get('/api/health/minimal')['health']['status'] def _get_pool(self, pool_name): pool = self._get("/api/pool/" + pool_name) @@ -133,8 +137,8 @@ class PoolTest(DashboardTestCase): """ pgp_prop = 'pg_placement_num' t = 0 - while (int(value) != pool[pgp_prop] or self._get('/api/health/minimal')['health']['status'] - != 'HEALTH_OK') and t < 180: + while (int(value) != pool[pgp_prop] or self._get_health_status() != 'HEALTH_OK') \ + and t < 180: time.sleep(2) t += 2 pool = self._get_pool(pool['pool_name']) @@ -158,6 +162,16 @@ class PoolTest(DashboardTestCase): self._delete('/api/pool/ddd') self.assertStatus(403) + def test_pool_configuration(self): + pool_name = 'device_health_metrics' + data = self._get('/api/pool/{}/configuration'.format(pool_name)) + self.assertStatus(200) + self.assertSchema(data, JList(JObj({ + 'name': str, + 'value': str, + 'source': int + }))) + def test_pool_list(self): data = self._get("/api/pool") self.assertStatus(200) @@ -313,23 +327,23 @@ class PoolTest(DashboardTestCase): with self.__yield_pool(pool_name): props = {'application_metadata': ['rbd', 'sth']} self._task_put('/api/pool/{}'.format(pool_name), props) - time.sleep(5) - self._validate_pool_properties(props, self._get_pool(pool_name)) + self._validate_pool_properties(props, self._get_pool(pool_name), + self.TIMEOUT_HEALTH_CLEAR * 2) properties = {'application_metadata': ['rgw']} self._task_put('/api/pool/' + pool_name, properties) - time.sleep(5) - self._validate_pool_properties(properties, self._get_pool(pool_name)) + self._validate_pool_properties(properties, self._get_pool(pool_name), + self.TIMEOUT_HEALTH_CLEAR * 2) properties = {'application_metadata': ['rbd', 'sth']} self._task_put('/api/pool/' + pool_name, properties) - time.sleep(5) - self._validate_pool_properties(properties, self._get_pool(pool_name)) + self._validate_pool_properties(properties, self._get_pool(pool_name), + self.TIMEOUT_HEALTH_CLEAR * 2) properties = {'application_metadata': ['rgw']} self._task_put('/api/pool/' + pool_name, properties) - time.sleep(5) - self._validate_pool_properties(properties, self._get_pool(pool_name)) + self._validate_pool_properties(properties, self._get_pool(pool_name), + self.TIMEOUT_HEALTH_CLEAR * 2) def test_pool_update_configuration(self): pool_name = 'pool_update_configuration' @@ -415,4 +429,5 @@ class PoolTest(DashboardTestCase): 'erasure_code_profiles': JList(JObj({}, allow_unknown=True)), 'used_rules': JObj({}, allow_unknown=True), 'used_profiles': JObj({}, allow_unknown=True), + 'nodes': JList(JObj({}, allow_unknown=True)), })) diff --git a/ceph/qa/tasks/mgr/dashboard/test_rbd.py b/ceph/qa/tasks/mgr/dashboard/test_rbd.py index 1c89651b4..48119383b 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rbd.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rbd.py @@ -5,7 +5,7 @@ from __future__ import absolute_import import time -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JLeaf, JList +from .helper import DashboardTestCase, JObj, JLeaf, JList class RbdTest(DashboardTestCase): @@ -173,14 +173,13 @@ class RbdTest(DashboardTestCase): cls._ceph_cmd(['osd', 'pool', 'delete', 'rbd_data', 'rbd_data', '--yes-i-really-really-mean-it']) - @classmethod - def create_image_in_trash(cls, pool, name, delay=0): - cls.create_image(pool, None, name, 10240) - img = cls._get('/api/block/image/{}%2F{}'.format(pool, name)) + def create_image_in_trash(self, pool, name, delay=0): + self.create_image(pool, None, name, 10240) + img = self._get('/api/block/image/{}%2F{}'.format(pool, name)) - cls._task_post("/api/block/image/{}%2F{}/move_trash".format(pool, name), + self._task_post("/api/block/image/{}%2F{}/move_trash".format(pool, name), {'delay': delay}) - + self.assertStatus([200, 201]) return img['id'] @classmethod @@ -236,6 +235,8 @@ class RbdTest(DashboardTestCase): 'block_name_prefix': JLeaf(str), 'name': JLeaf(str), 'id': JLeaf(str), + 'unique_id': JLeaf(str), + 'image_format': JLeaf(int), 'pool_name': JLeaf(str), 'namespace': JLeaf(str, none=True), 'features': JLeaf(int), @@ -774,7 +775,6 @@ class RbdTest(DashboardTestCase): def test_move_image_to_trash(self): id = self.create_image_in_trash('rbd', 'test_rbd') - self.assertStatus(200) self.get_image('rbd', None, 'test_rbd') self.assertStatus(404) diff --git a/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py b/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py index f8268f352..39e5f895f 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py @@ -3,7 +3,7 @@ from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase class RbdMirroringTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_requests.py b/ceph/qa/tasks/mgr/dashboard/test_requests.py index 22376c0a2..0d9f8d9ba 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_requests.py +++ b/ceph/qa/tasks/mgr/dashboard/test_requests.py @@ -2,7 +2,7 @@ from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase class RequestsTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_rgw.py b/ceph/qa/tasks/mgr/dashboard/test_rgw.py index 3e5b73599..1e707c33d 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rgw.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rgw.py @@ -10,7 +10,7 @@ from cryptography.hazmat.primitives.twofactor.totp import TOTP from cryptography.hazmat.primitives.hashes import SHA1 from six.moves.urllib import parse -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList, JLeaf +from .helper import DashboardTestCase, JObj, JList, JLeaf logger = logging.getLogger(__name__) @@ -510,6 +510,11 @@ class RgwUserTest(RgwTestCase): self.assertGreaterEqual(len(data), 1) self.assertIn('admin', data) + def test_get_emails(self): + data = self._get('/api/rgw/user/get_emails') + self.assertStatus(200) + self.assertSchema(data, JList(str)) + def test_create_get_update_delete(self): # Create a new user. self._post('/api/rgw/user', params={ diff --git a/ceph/qa/tasks/mgr/dashboard/test_role.py b/ceph/qa/tasks/mgr/dashboard/test_role.py index d678fa195..dbfaea9e4 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_role.py +++ b/ceph/qa/tasks/mgr/dashboard/test_role.py @@ -2,7 +2,7 @@ from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase class RoleTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_settings.py b/ceph/qa/tasks/mgr/dashboard/test_settings.py index 4c6ebeaca..2d890484a 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_settings.py +++ b/ceph/qa/tasks/mgr/dashboard/test_settings.py @@ -2,7 +2,7 @@ from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase, JList, JObj, JAny +from .helper import DashboardTestCase, JList, JObj, JAny class SettingsTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_summary.py b/ceph/qa/tasks/mgr/dashboard/test_summary.py index 808107a31..a31f89146 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_summary.py +++ b/ceph/qa/tasks/mgr/dashboard/test_summary.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from tasks.mgr.dashboard.helper import DashboardTestCase +from .helper import DashboardTestCase class SummaryTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_user.py b/ceph/qa/tasks/mgr/dashboard/test_user.py index f3606ed85..ea7beee6d 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_user.py +++ b/ceph/qa/tasks/mgr/dashboard/test_user.py @@ -6,7 +6,7 @@ import time from datetime import datetime, timedelta -from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JLeaf +from .helper import DashboardTestCase, JObj, JLeaf class UserTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/test_orchestrator_cli.py b/ceph/qa/tasks/mgr/test_orchestrator_cli.py index db9226323..ed0c52fea 100644 --- a/ceph/qa/tasks/mgr/test_orchestrator_cli.py +++ b/ceph/qa/tasks/mgr/test_orchestrator_cli.py @@ -163,8 +163,10 @@ data_devices: self._orch_cmd("apply", "nfs", "service_name", "2") def test_error(self): - ret = self._orch_cmd_result("host", "add", "raise_no_support") - self.assertEqual(ret, errno.ENOENT) + ret = self._orch_cmd_result("host", "add", "raise_validation_error") + self.assertEqual(ret, errno.EINVAL) + ret = self._orch_cmd_result("host", "add", "raise_error") + self.assertEqual(ret, errno.EINVAL) ret = self._orch_cmd_result("host", "add", "raise_bug") self.assertEqual(ret, errno.EINVAL) ret = self._orch_cmd_result("host", "add", "raise_not_implemented") diff --git a/ceph/qa/tasks/ragweed.py b/ceph/qa/tasks/ragweed.py index 052992a49..107dc3d68 100644 --- a/ceph/qa/tasks/ragweed.py +++ b/ceph/qa/tasks/ragweed.py @@ -18,6 +18,33 @@ from teuthology.orchestra import run log = logging.getLogger(__name__) + +def get_ragweed_branches(config, client_conf): + """ + figure out the ragweed branch according to the per-client settings + + use force-branch is specified, and fall back to the ones deduced using ceph + branch under testing + """ + force_branch = client_conf.get('force-branch', None) + if force_branch: + return [force_branch] + else: + S3_BRANCHES = ['master', 'nautilus', 'mimic', + 'luminous', 'kraken', 'jewel'] + ceph_branch = config.get('branch') + suite_branch = config.get('suite_branch', ceph_branch) + if suite_branch in S3_BRANCHES: + branch = client_conf.get('branch', 'ceph-' + suite_branch) + else: + branch = client_conf.get('branch', suite_branch) + default_branch = client_conf.get('default-branch', None) + if default_branch: + return [branch, default_branch] + else: + return [branch] + + @contextlib.contextmanager def download(ctx, config): """ @@ -30,46 +57,21 @@ def download(ctx, config): assert isinstance(config, dict) log.info('Downloading ragweed...') testdir = teuthology.get_testdir(ctx) - s3_branches = [ 'master', 'nautilus', 'mimic', 'luminous', 'kraken', 'jewel' ] for (client, cconf) in config.items(): - default_branch = '' - branch = cconf.get('force-branch', None) - if not branch: - default_branch = cconf.get('default-branch', None) - ceph_branch = ctx.config.get('branch') - suite_branch = ctx.config.get('suite_branch', ceph_branch) - ragweed_repo = ctx.config.get('ragweed_repo', teuth_config.ceph_git_base_url + 'ragweed.git') - if suite_branch in s3_branches: - branch = cconf.get('branch', 'ceph-' + suite_branch) - else: - branch = cconf.get('branch', suite_branch) - if not branch: - raise ValueError( - "Could not determine what branch to use for ragweed!") - else: + ragweed_repo = ctx.config.get('ragweed_repo', + teuth_config.ceph_git_base_url + 'ragweed.git') + for branch in get_ragweed_branches(ctx.config, cconf): log.info("Using branch '%s' for ragweed", branch) - sha1 = cconf.get('sha1') - try: - ctx.cluster.only(client).run( - args=[ - 'git', 'clone', - '-b', branch, - ragweed_repo, - '{tdir}/ragweed'.format(tdir=testdir), - ], - ) - except Exception as e: - if not default_branch: - raise e - ctx.cluster.only(client).run( - args=[ - 'git', 'clone', - '-b', default_branch, - ragweed_repo, - '{tdir}/ragweed'.format(tdir=testdir), - ], - ) + try: + ctx.cluster.only(client).sh( + script=f'git clone -b {branch} {ragweed_repo} {testdir}/ragweed') + break + except Exception as e: + exc = e + else: + raise exc + sha1 = cconf.get('sha1') if sha1 is not None: ctx.cluster.only(client).run( args=[ diff --git a/ceph/qa/tasks/vstart_runner.py b/ceph/qa/tasks/vstart_runner.py index a248db9e4..a7f976d8e 100644 --- a/ceph/qa/tasks/vstart_runner.py +++ b/ceph/qa/tasks/vstart_runner.py @@ -30,8 +30,8 @@ Alternative usage: """ -from six import StringIO from io import BytesIO +from io import StringIO from collections import defaultdict import getpass import signal @@ -152,8 +152,8 @@ else: def rm_nonascii_chars(var): - var = var.replace('\xe2\x80\x98', '\'') - var = var.replace('\xe2\x80\x99', '\'') + var = var.replace(b'\xe2\x80\x98', b'\'') + var = var.replace(b'\xe2\x80\x99', b'\'') return var class LocalRemoteProcess(object): @@ -177,8 +177,14 @@ class LocalRemoteProcess(object): out, err = self.subproc.communicate() out, err = rm_nonascii_chars(out), rm_nonascii_chars(err) - self.stdout.write(out) - self.stderr.write(err) + if isinstance(self.stdout, StringIO): + self.stdout.write(out.decode(errors='ignore')) + else: + self.stdout.write(out) + if isinstance(self.stderr, StringIO): + self.stderr.write(err.decode(errors='ignore')) + else: + self.stderr.write(err) self.exitstatus = self.returncode = self.subproc.returncode @@ -379,12 +385,12 @@ class LocalRemote(object): env=env) if stdin: - if not isinstance(stdin, six.string_types): + if not isinstance(stdin, str): raise RuntimeError("Can't handle non-string stdins on a vstart cluster") # Hack: writing to stdin is not deadlock-safe, but it "always" works # as long as the input buffer is "small" - subproc.stdin.write(stdin) + subproc.stdin.write(stdin.encode()) proc = LocalRemoteProcess( args, subproc, check_status, @@ -396,7 +402,8 @@ class LocalRemote(object): return proc - # XXX: for compatibility keep this method same teuthology.orchestra.remote.sh + # XXX: for compatibility keep this method same as teuthology.orchestra.remote.sh + # BytesIO is being used just to keep things identical def sh(self, script, **kwargs): """ Shortcut for run method. @@ -405,13 +412,18 @@ class LocalRemote(object): my_name = remote.sh('whoami') remote_date = remote.sh('date') """ + from io import BytesIO + if 'stdout' not in kwargs: - kwargs['stdout'] = StringIO() + kwargs['stdout'] = BytesIO() if 'args' not in kwargs: kwargs['args'] = script proc = self.run(**kwargs) - return proc.stdout.getvalue() - + out = proc.stdout.getvalue() + if isinstance(out, bytes): + return out.decode() + else: + return out class LocalDaemon(object): def __init__(self, daemon_type, daemon_id): @@ -1288,6 +1300,7 @@ def exec_test(): global opt_log_ps_output opt_log_ps_output = False use_kernel_client = False + opt_verbose = True args = sys.argv[1:] flags = [a for a in args if a.startswith("-")] @@ -1309,6 +1322,8 @@ def exec_test(): clear_old_log() elif f == "--kclient": use_kernel_client = True + elif '--no-verbose' == f: + opt_verbose = False else: log.error("Unknown option '{0}'".format(f)) sys.exit(-1) @@ -1330,7 +1345,8 @@ def exec_test(): # Tolerate no MDSs or clients running at start ps_txt = six.ensure_str(remote.run( - args=["ps", "-u"+str(os.getuid())] + args=["ps", "-u"+str(os.getuid())], + stdout=StringIO() ).stdout.getvalue().strip()) lines = ps_txt.split("\n")[1:] for line in lines: @@ -1350,11 +1366,17 @@ def exec_test(): vstart_env["OSD"] = "4" vstart_env["MGR"] = max(max_required_mgr, 1).__str__() - args = [os.path.join(SRC_PREFIX, "vstart.sh"), "-n", "-d", - "--nolockdep"] + args = [ + os.path.join(SRC_PREFIX, "vstart.sh"), + "-n", + "--nolockdep", + ] if require_memstore: args.append("--memstore") + if opt_verbose: + args.append("-d") + # usually, i get vstart.sh running completely in less than 100 # seconds. remote.run(args=args, env=vstart_env, timeout=(3 * 60)) @@ -1390,7 +1412,7 @@ def exec_test(): "mds", "allow", "mon", "allow r"]) - open("./keyring", "a").write(p.stdout.getvalue()) + open("./keyring", "ab").write(p.stdout.getvalue()) if use_kernel_client: mount = LocalKernelMount(ctx, test_dir, client_id) diff --git a/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh b/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh index 67750cd0d..da96ccd32 100755 --- a/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh +++ b/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh @@ -17,6 +17,13 @@ awscli_dir=${HOME}/awscli_temp export PATH=${PATH}:${awscli_dir} rgw_host=$(hostname --fqdn) +if echo "$rgw_host" | grep -q '\.' ; then + : +else + host_domain=".front.sepia.ceph.com" + echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain" + rgw_host="${rgw_host}${host_domain}" +fi rgw_port=80 echo "Fully Qualified Domain Name: $rgw_host" diff --git a/ceph/selinux/ceph.te b/ceph/selinux/ceph.te index 81b4d0067..77d35d971 100644 --- a/ceph/selinux/ceph.te +++ b/ceph/selinux/ceph.te @@ -13,11 +13,14 @@ require { type urandom_device_t; type setfiles_t; type nvme_device_t; + type targetd_etc_rw_t; + type amqp_port_t; + type soundd_port_t; class sock_file unlink; class tcp_socket name_connect_t; class lnk_file { create getattr read unlink }; class dir { add_name create getattr open read remove_name rmdir search write }; - class file { create getattr open read rename unlink write }; + class file { create getattr open read rename unlink write ioctl }; class blk_file { getattr ioctl open read write }; class capability2 block_suspend; class process2 { nnp_transition nosuid_transition }; @@ -87,6 +90,8 @@ corenet_tcp_sendrecv_cyphesis_port(ceph_t) allow ceph_t commplex_main_port_t:tcp_socket name_connect; allow ceph_t http_cache_port_t:tcp_socket name_connect; +allow ceph_t amqp_port_t:tcp_socket name_connect; +allow ceph_t soundd_port_t:tcp_socket name_connect; corecmd_exec_bin(ceph_t) corecmd_exec_shell(ceph_t) @@ -137,7 +142,7 @@ allow ceph_t sysfs_t:file { read getattr open }; allow ceph_t sysfs_t:lnk_file { read getattr }; allow ceph_t configfs_t:dir { add_name create getattr open read remove_name rmdir search write }; -allow ceph_t configfs_t:file { getattr open read write }; +allow ceph_t configfs_t:file { getattr open read write ioctl }; allow ceph_t configfs_t:lnk_file { create getattr read unlink }; @@ -150,6 +155,8 @@ allow ceph_t var_run_t:file { read write create open getattr }; allow ceph_t init_var_run_t:file getattr; allow init_t ceph_t:process2 { nnp_transition nosuid_transition }; +allow ceph_t targetd_etc_rw_t:dir { getattr search }; + fsadm_manage_pid(ceph_t) #============= setfiles_t ============== diff --git a/ceph/src/.git_version b/ceph/src/.git_version index ed8078796..cd1d01c51 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -7447c15c6ff58d7fce91843b705a268a1917325c -15.2.4 +2c93eff00150f0cc5f106a559557a58d3d7b6f1f +15.2.5 diff --git a/ceph/src/ceph-volume/ceph_volume/api/lvm.py b/ceph/src/ceph-volume/ceph_volume/api/lvm.py index b8ea78bea..0e6f42dac 100644 --- a/ceph/src/ceph-volume/ceph_volume/api/lvm.py +++ b/ceph/src/ceph-volume/ceph_volume/api/lvm.py @@ -9,14 +9,66 @@ import uuid from itertools import repeat from math import floor from ceph_volume import process, util -from ceph_volume.exceptions import ( - MultipleLVsError, MultipleVGsError, - MultiplePVsError, SizeAllocationError -) +from ceph_volume.exceptions import SizeAllocationError logger = logging.getLogger(__name__) +def convert_filters_to_str(filters): + """ + Convert filter args from dictionary to following format - + filters={filter_name=filter_val,...} + """ + if not filters: + return filters + + filter_arg = '' + for k, v in filters.items(): + filter_arg += k + '=' + v + ',' + # get rid of extra comma at the end + filter_arg = filter_arg[:len(filter_arg) - 1] + + return filter_arg + + +def convert_tags_to_str(tags): + """ + Convert tags from dictionary to following format - + tags={tag_name=tag_val,...} + """ + if not tags: + return tags + + tag_arg = 'tags={' + for k, v in tags.items(): + tag_arg += k + '=' + v + ',' + # get rid of extra comma at the end + tag_arg = tag_arg[:len(tag_arg) - 1] + '}' + + return tag_arg + + +def make_filters_lvmcmd_ready(filters, tags): + """ + Convert filters (including tags) from dictionary to following format - + filter_name=filter_val...,tags={tag_name=tag_val,...} + + The command will look as follows = + lvs -S filter_name=filter_val...,tags={tag_name=tag_val,...} + """ + filters = convert_filters_to_str(filters) + tags = convert_tags_to_str(tags) + + if filters and tags: + return filters + ',' + tags + if filters and not tags: + return filters + if not filters and tags: + return tags + else: + return '' + + def _output_parser(output, fields): """ Newer versions of LVM allow ``--reportformat=json``, but older versions, @@ -289,29 +341,6 @@ def is_ceph_device(lv): PV_FIELDS = 'pv_name,pv_tags,pv_uuid,vg_name,lv_uuid' -def get_api_pvs(): - """ - Return the list of physical volumes configured for lvm and available in the - system using flags to include common metadata associated with them like the uuid - - This will only return physical volumes set up to work with LVM. - - Command and delimited output should look like:: - - $ pvs --noheadings --readonly --separator=';' -o pv_name,pv_tags,pv_uuid - /dev/sda1;; - /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D - - """ - stdout, stderr, returncode = process.call( - ['pvs', '--no-heading', '--readonly', '--separator=";"', '-o', - PV_FIELDS], - verbose_on_failure=False - ) - - return _output_parser(stdout, PV_FIELDS) - - class PVolume(object): """ Represents a Physical Volume from LVM, with some top-level attributes like @@ -347,7 +376,8 @@ class PVolume(object): self.set_tag(k, v) # after setting all the tags, refresh them for the current object, use the # pv_* identifiers to filter because those shouldn't change - pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid) + pv_object = self.get_first_pv(filter={'pv_name': self.pv_name, + 'pv_uuid': self.pv_uuid}) self.tags = pv_object.tags def set_tag(self, key, value): @@ -375,100 +405,6 @@ class PVolume(object): ) -class PVolumes(list): - """ - A list of all known (physical) volumes for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self, populate=True): - if populate: - self._populate() - - def _populate(self): - # get all the pvs in the current system - for pv_item in get_api_pvs(): - self.append(PVolume(**pv_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - """ - filtered = [i for i in self] - if pv_name: - filtered = [i for i in filtered if i.pv_name == pv_name] - - if pv_uuid: - filtered = [i for i in filtered if i.pv_uuid == pv_uuid] - - # at this point, `filtered` has either all the physical volumes in self - # or is an actual filtered list if any filters were applied - if pv_tags: - tag_filtered = [] - for pvolume in filtered: - matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items()) - if matches: - tag_filtered.append(pvolume) - # return the tag_filtered pvolumes here, the `filtered` list is no - # longer usable - return tag_filtered - - return filtered - - def filter(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - Filter out volumes on top level attributes like ``pv_name`` or by - ``pv_tags`` where a dict is required. For example, to find a physical - volume that has an OSD ID of 0, the filter would look like:: - - pv_tags={'ceph.osd_id': '0'} - - """ - if not any([pv_name, pv_uuid, pv_tags]): - raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags' - '(none given)') - - filtered_pvs = PVolumes(populate=False) - filtered_pvs.extend(self._filter(pv_name, pv_uuid, pv_tags)) - return filtered_pvs - - def get(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple pvs are matched - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple pvs being found, since a lot of metadata - is shared between pvs of a distinct OSD. - """ - if not any([pv_name, pv_uuid, pv_tags]): - return None - pvs = self._filter( - pv_name=pv_name, - pv_uuid=pv_uuid, - pv_tags=pv_tags - ) - if not pvs: - return None - if len(pvs) > 1 and pv_tags: - raise MultiplePVsError(pv_name) - return pvs[0] - - def create_pv(device): """ Create a physical volume from a device, useful when devices need to be later mapped @@ -510,18 +446,40 @@ def remove_pv(pv_name): ) -def get_pv(pv_name=None, pv_uuid=None, pv_tags=None, pvs=None): +def get_pvs(fields=PV_FIELDS, filters='', tags=None): """ - Return a matching pv (physical volume) for the current system, requiring - ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one - pv is found. + Return a list of PVs that are available on the system and match the + filters and tags passed. Argument filters takes a dictionary containing + arguments required by -S option of LVM. Passing a list of LVM tags can be + quite tricky to pass as a dictionary within dictionary, therefore pass + dictionary of tags via tags argument and tricky part will be taken care of + by the helper methods. + + :param fields: string containing list of fields to be displayed by the + pvs command + :param sep: string containing separator to be used between two fields + :param filters: dictionary containing LVM filters + :param tags: dictionary containng LVM tags + :returns: list of class PVolume object representing pvs on the system """ - if not any([pv_name, pv_uuid, pv_tags]): - return None - if pvs is None or len(pvs) == 0: - pvs = PVolumes() + filters = make_filters_lvmcmd_ready(filters, tags) + args = ['pvs', '--no-heading', '--readonly', '--separator=";"', '-S', + filters, '-o', fields] + + stdout, stderr, returncode = process.call(args, verbose_on_failure=False) + pvs_report = _output_parser(stdout, fields) + return [PVolume(**pv_report) for pv_report in pvs_report] + - return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags) +def get_first_pv(fields=PV_FIELDS, filters=None, tags=None): + """ + Wrapper of get_pv meant to be a convenience method to avoid the phrase:: + pvs = get_pvs() + if len(pvs) >= 1: + pv = pvs[0] + """ + pvs = get_pvs(fields=fields, filters=filters, tags=tags) + return pvs[0] if len(pvs) > 0 else [] ################################ @@ -534,27 +492,6 @@ VG_FIELDS = 'vg_name,pv_count,lv_count,vg_attr,vg_extent_count,vg_free_count,vg_ VG_CMD_OPTIONS = ['--noheadings', '--readonly', '--units=b', '--nosuffix', '--separator=";"'] -def get_api_vgs(): - """ - Return the list of group volumes available in the system using flags to - include common metadata associated with them - - Command and sample delimited output should look like:: - - $ vgs --noheadings --units=b --readonly --separator=';' \ - -o vg_name,pv_count,lv_count,vg_attr,vg_free_count,vg_extent_size - ubuntubox-vg;1;2;wz--n-;12; - - To normalize sizing, the units are forced in 'g' which is equivalent to - gigabytes, which uses multiples of 1024 (as opposed to 1000) - """ - stdout, stderr, returncode = process.call( - ['vgs'] + VG_CMD_OPTIONS + ['-o', VG_FIELDS], - verbose_on_failure=False - ) - return _output_parser(stdout, VG_FIELDS) - - class VolumeGroup(object): """ Represents an LVM group, with some top-level attributes like ``vg_name`` @@ -655,98 +592,6 @@ class VolumeGroup(object): return int(int(self.vg_free_count) / slots) -class VolumeGroups(list): - """ - A list of all known volume groups for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self, populate=True): - if populate: - self._populate() - - def _populate(self): - # get all the vgs in the current system - for vg_item in get_api_vgs(): - self.append(VolumeGroup(**vg_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, vg_name=None, vg_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - - .. note:: ``vg_tags`` is not yet implemented - """ - filtered = [i for i in self] - if vg_name: - filtered = [i for i in filtered if i.vg_name == vg_name] - - # at this point, `filtered` has either all the volumes in self or is an - # actual filtered list if any filters were applied - if vg_tags: - tag_filtered = [] - for volume in filtered: - matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items()) - if matches: - tag_filtered.append(volume) - return tag_filtered - - return filtered - - def filter(self, vg_name=None, vg_tags=None): - """ - Filter out groups on top level attributes like ``vg_name`` or by - ``vg_tags`` where a dict is required. For example, to find a Ceph group - with dmcache as the type, the filter would look like:: - - vg_tags={'ceph.type': 'dmcache'} - - .. warning:: These tags are not documented because they are currently - unused, but are here to maintain API consistency - """ - if not any([vg_name, vg_tags]): - raise TypeError('.filter() requires vg_name or vg_tags (none given)') - - filtered_vgs = VolumeGroups(populate=False) - filtered_vgs.extend(self._filter(vg_name, vg_tags)) - return filtered_vgs - - def get(self, vg_name=None, vg_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple VGs are matched - - It is useful to use ``tags`` when trying to find a specific volume group, - but it can also lead to multiple vgs being found (although unlikely) - """ - if not any([vg_name, vg_tags]): - return None - vgs = self._filter( - vg_name=vg_name, - vg_tags=vg_tags - ) - if not vgs: - return None - if len(vgs) > 1: - # this is probably never going to happen, but it is here to keep - # the API code consistent - raise MultipleVGsError(vg_name) - return vgs[0] - - def create_vg(devices, name=None, name_prefix=None): """ Create a Volume Group. Command looks like:: @@ -776,8 +621,7 @@ def create_vg(devices, name=None, name_prefix=None): name] + devices ) - vg = get_vg(vg_name=name) - return vg + return get_first_vg(filters={'vg_name': name}) def extend_vg(vg, devices): @@ -801,8 +645,7 @@ def extend_vg(vg, devices): vg.name] + devices ) - vg = get_vg(vg_name=vg.name) - return vg + return get_first_vg(filters={'vg_name': vg.name}) def reduce_vg(vg, devices): @@ -824,8 +667,7 @@ def reduce_vg(vg, devices): vg.name] + devices ) - vg = get_vg(vg_name=vg.name) - return vg + return get_first_vg(filter={'vg_name': vg.name}) def remove_vg(vg_name): @@ -847,20 +689,39 @@ def remove_vg(vg_name): ) -def get_vg(vg_name=None, vg_tags=None, vgs=None): +def get_vgs(fields=VG_FIELDS, filters='', tags=None): """ - Return a matching vg for the current system, requires ``vg_name`` or - ``tags``. Raises an error if more than one vg is found. + Return a list of VGs that are available on the system and match the + filters and tags passed. Argument filters takes a dictionary containing + arguments required by -S option of LVM. Passing a list of LVM tags can be + quite tricky to pass as a dictionary within dictionary, therefore pass + dictionary of tags via tags argument and tricky part will be taken care of + by the helper methods. - It is useful to use ``tags`` when trying to find a specific volume group, - but it can also lead to multiple vgs being found. + :param fields: string containing list of fields to be displayed by the + vgs command + :param sep: string containing separator to be used between two fields + :param filters: dictionary containing LVM filters + :param tags: dictionary containng LVM tags + :returns: list of class VolumeGroup object representing vgs on the system """ - if not any([vg_name, vg_tags]): - return None - if vgs is None or len(vgs) == 0: - vgs = VolumeGroups() + filters = make_filters_lvmcmd_ready(filters, tags) + args = ['vgs'] + VG_CMD_OPTIONS + ['-S', filters, '-o', fields] - return vgs.get(vg_name=vg_name, vg_tags=vg_tags) + stdout, stderr, returncode = process.call(args, verbose_on_failure=False) + vgs_report =_output_parser(stdout, fields) + return [VolumeGroup(**vg_report) for vg_report in vgs_report] + + +def get_first_vg(fields=VG_FIELDS, filters=None, tags=None): + """ + Wrapper of get_vg meant to be a convenience method to avoid the phrase:: + vgs = get_vgs() + if len(vgs) >= 1: + vg = vgs[0] + """ + vgs = get_vgs(fields=fields, filters=filters, tags=tags) + return vgs[0] if len(vgs) > 0 else [] def get_device_vgs(device, name_prefix=''): @@ -881,24 +742,6 @@ def get_device_vgs(device, name_prefix=''): LV_FIELDS = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size' LV_CMD_OPTIONS = ['--noheadings', '--readonly', '--separator=";"', '-a'] -def get_api_lvs(): - """ - Return the list of logical volumes available in the system using flags to include common - metadata associated with them - - Command and delimited output should look like:: - - $ lvs --noheadings --readonly --separator=';' -a -o lv_tags,lv_path,lv_name,vg_name - ;/dev/ubuntubox-vg/root;root;ubuntubox-vg - ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg - - """ - stdout, stderr, returncode = process.call( - ['lvs'] + LV_CMD_OPTIONS + ['-o', LV_FIELDS], - verbose_on_failure=False - ) - return _output_parser(stdout, LV_FIELDS) - class Volume(object): """ @@ -1025,113 +868,6 @@ class Volume(object): process.call(['lvchange', '-an', self.lv_path]) -class Volumes(list): - """ - A list of all known (logical) volumes for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self): - self._populate() - - def _populate(self): - # get all the lvs in the current system - for lv_item in get_api_lvs(): - self.append(Volume(**lv_item)) - - def _purge(self): - """ - Delete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - """ - filtered = [i for i in self] - if lv_name: - filtered = [i for i in filtered if i.lv_name == lv_name] - - if vg_name: - filtered = [i for i in filtered if i.vg_name == vg_name] - - if lv_uuid: - filtered = [i for i in filtered if i.lv_uuid == lv_uuid] - - if lv_path: - filtered = [i for i in filtered if i.lv_path == lv_path] - - # at this point, `filtered` has either all the volumes in self or is an - # actual filtered list if any filters were applied - if lv_tags: - tag_filtered = [] - for volume in filtered: - # all the tags we got need to match on the volume - matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items()) - if matches: - tag_filtered.append(volume) - return tag_filtered - - return filtered - - def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - Filter out volumes on top level attributes like ``lv_name`` or by - ``lv_tags`` where a dict is required. For example, to find a volume - that has an OSD ID of 0, the filter would look like:: - - lv_tags={'ceph.osd_id': '0'} - - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)') - # first find the filtered volumes with the values in self - filtered_volumes = self._filter( - lv_name=lv_name, - vg_name=vg_name, - lv_path=lv_path, - lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - # then purge everything - self._purge() - # and add the filtered items - self.extend(filtered_volumes) - - def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple LVs are matched - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple lvs being found, since a lot of metadata - is shared between lvs of a distinct OSD. - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - return None - lvs = self._filter( - lv_name=lv_name, - vg_name=vg_name, - lv_path=lv_path, - lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - if not lvs: - return None - if len(lvs) > 1: - raise MultipleLVsError(lv_name, lv_path) - return lvs[0] - - def create_lv(name_prefix, uuid, vg=None, @@ -1204,7 +940,7 @@ def create_lv(name_prefix, ] process.run(command) - lv = get_lv(lv_name=name, vg_name=vg.vg_name) + lv = get_first_lv(filters={'lv_name': name, 'vg_name': vg.vg_name}) if tags is None: tags = { @@ -1233,104 +969,6 @@ def create_lv(name_prefix, return lv -def remove_lv(lv): - """ - Removes a logical volume given it's absolute path. - - Will return True if the lv is successfully removed or - raises a RuntimeError if the removal fails. - - :param lv: A ``Volume`` object or the path for an LV - """ - if isinstance(lv, Volume): - path = lv.lv_path - else: - path = lv - - stdout, stderr, returncode = process.call( - [ - 'lvremove', - '-v', # verbose - '-f', # force it - path - ], - show_command=True, - terminal_verbose=True, - ) - if returncode != 0: - raise RuntimeError("Unable to remove %s" % path) - return True - - -def is_lv(dev, lvs=None): - """ - Boolean to detect if a device is an LV or not. - """ - splitname = dmsetup_splitname(dev) - # Allowing to optionally pass `lvs` can help reduce repetitive checks for - # multiple devices at once. - if lvs is None or len(lvs) == 0: - lvs = Volumes() - - if splitname.get('LV_NAME'): - lvs.filter(lv_name=splitname['LV_NAME'], vg_name=splitname['VG_NAME']) - return len(lvs) > 0 - return False - -def get_lv_by_name(name): - stdout, stderr, returncode = process.call( - ['lvs', '--noheadings', '-o', LV_FIELDS, '-S', - 'lv_name={}'.format(name)], - verbose_on_failure=False - ) - lvs = _output_parser(stdout, LV_FIELDS) - return [Volume(**lv) for lv in lvs] - -def get_lvs_by_tag(lv_tag): - stdout, stderr, returncode = process.call( - ['lvs', '--noheadings', '--separator=";"', '-a', '-o', LV_FIELDS, '-S', - 'lv_tags={{{}}}'.format(lv_tag)], - verbose_on_failure=False - ) - lvs = _output_parser(stdout, LV_FIELDS) - return [Volume(**lv) for lv in lvs] - -def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None, lvs=None): - """ - Return a matching lv for the current system, requiring ``lv_name``, - ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv - is found. - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple lvs being found, since a lot of metadata - is shared between lvs of a distinct OSD. - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - return None - if lvs is None: - lvs = Volumes() - return lvs.get( - lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - - -def get_lv_from_argument(argument): - """ - Helper proxy function that consumes a possible logical volume passed in from the CLI - in the form of `vg/lv`, but with some validation so that an argument that is a full - path to a device can be ignored - """ - if argument.startswith('/'): - lv = get_lv(lv_path=argument) - return lv - try: - vg_name, lv_name = argument.split('/') - except (ValueError, AttributeError): - return None - return get_lv(lv_name=lv_name, vg_name=vg_name) - - def create_lvs(volume_group, parts=None, size=None, name_prefix='ceph-lv'): """ Create multiple Logical Volumes from a Volume Group by calculating the @@ -1374,141 +1012,34 @@ def create_lvs(volume_group, parts=None, size=None, name_prefix='ceph-lv'): return lvs -def get_device_lvs(device, name_prefix=''): - stdout, stderr, returncode = process.call( - ['pvs'] + LV_CMD_OPTIONS + ['-o', LV_FIELDS, device], - verbose_on_failure=False - ) - lvs = _output_parser(stdout, LV_FIELDS) - return [Volume(**lv) for lv in lvs if lv['lv_name'] and - lv['lv_name'].startswith(name_prefix)] - - -############################################################# -# -# New methods to get PVs, LVs, and VGs. -# Later, these can be easily merged with get_api_* methods -# -########################################################### - -def convert_filters_to_str(filters): - """ - Convert filter args from dictionary to following format - - filters={filter_name=filter_val,...} - """ - if not filters: - return filters - - filter_arg = '' - for k, v in filters.items(): - filter_arg += k + '=' + v + ',' - # get rid of extra comma at the end - filter_arg = filter_arg[:len(filter_arg) - 1] - - return filter_arg - -def convert_tags_to_str(tags): - """ - Convert tags from dictionary to following format - - tags={tag_name=tag_val,...} +def remove_lv(lv): """ - if not tags: - return tags - - tag_arg = 'tags={' - for k, v in tags.items(): - tag_arg += k + '=' + v + ',' - # get rid of extra comma at the end - tag_arg = tag_arg[:len(tag_arg) - 1] + '}' - - return tag_arg + Removes a logical volume given it's absolute path. -def make_filters_lvmcmd_ready(filters, tags): - """ - Convert filters (including tags) from dictionary to following format - - filter_name=filter_val...,tags={tag_name=tag_val,...} + Will return True if the lv is successfully removed or + raises a RuntimeError if the removal fails. - The command will look as follows = - lvs -S filter_name=filter_val...,tags={tag_name=tag_val,...} + :param lv: A ``Volume`` object or the path for an LV """ - filters = convert_filters_to_str(filters) - tags = convert_tags_to_str(tags) - - if filters and tags: - return filters + ',' + tags - if filters and not tags: - return filters - if not filters and tags: - return tags + if isinstance(lv, Volume): + path = lv.lv_path else: - return '' - -def get_pvs(fields=PV_FIELDS, filters='', tags=None): - """ - Return a list of PVs that are available on the system and match the - filters and tags passed. Argument filters takes a dictionary containing - arguments required by -S option of LVM. Passing a list of LVM tags can be - quite tricky to pass as a dictionary within dictionary, therefore pass - dictionary of tags via tags argument and tricky part will be taken care of - by the helper methods. - - :param fields: string containing list of fields to be displayed by the - pvs command - :param sep: string containing separator to be used between two fields - :param filters: dictionary containing LVM filters - :param tags: dictionary containng LVM tags - :returns: list of class PVolume object representing pvs on the system - """ - filters = make_filters_lvmcmd_ready(filters, tags) - args = ['pvs', '--no-heading', '--readonly', '--separator=";"', '-S', - filters, '-o', fields] - - stdout, stderr, returncode = process.call(args, verbose_on_failure=False) - pvs_report = _output_parser(stdout, fields) - return [PVolume(**pv_report) for pv_report in pvs_report] - -def get_first_pv(fields=PV_FIELDS, filters=None, tags=None): - """ - Wrapper of get_pv meant to be a convenience method to avoid the phrase:: - pvs = get_pvs() - if len(pvs) >= 1: - pv = pvs[0] - """ - pvs = get_pvs(fields=fields, filters=filters, tags=tags) - return pvs[0] if len(pvs) > 0 else [] - -def get_vgs(fields=VG_FIELDS, filters='', tags=None): - """ - Return a list of VGs that are available on the system and match the - filters and tags passed. Argument filters takes a dictionary containing - arguments required by -S option of LVM. Passing a list of LVM tags can be - quite tricky to pass as a dictionary within dictionary, therefore pass - dictionary of tags via tags argument and tricky part will be taken care of - by the helper methods. - - :param fields: string containing list of fields to be displayed by the - vgs command - :param sep: string containing separator to be used between two fields - :param filters: dictionary containing LVM filters - :param tags: dictionary containng LVM tags - :returns: list of class VolumeGroup object representing vgs on the system - """ - filters = make_filters_lvmcmd_ready(filters, tags) - args = ['vgs'] + VG_CMD_OPTIONS + ['-S', filters, '-o', fields] + path = lv - stdout, stderr, returncode = process.call(args, verbose_on_failure=False) - vgs_report =_output_parser(stdout, fields) - return [VolumeGroup(**vg_report) for vg_report in vgs_report] + stdout, stderr, returncode = process.call( + [ + 'lvremove', + '-v', # verbose + '-f', # force it + path + ], + show_command=True, + terminal_verbose=True, + ) + if returncode != 0: + raise RuntimeError("Unable to remove %s" % path) + return True -def get_first_vg(fields=VG_FIELDS, filters=None, tags=None): - """ - Wrapper of get_vg meant to be a convenience method to avoid the phrase:: - vgs = get_vgs() - if len(vgs) >= 1: - vg = vgs[0] - """ - vgs = get_vgs(fields=fields, filters=filters, tags=tags) - return vgs[0] if len(vgs) > 0 else [] def get_lvs(fields=LV_FIELDS, filters='', tags=None): """ @@ -1533,6 +1064,7 @@ def get_lvs(fields=LV_FIELDS, filters='', tags=None): lvs_report = _output_parser(stdout, fields) return [Volume(**lv_report) for lv_report in lvs_report] + def get_first_lv(fields=LV_FIELDS, filters=None, tags=None): """ Wrapper of get_lv meant to be a convenience method to avoid the phrase:: @@ -1542,3 +1074,33 @@ def get_first_lv(fields=LV_FIELDS, filters=None, tags=None): """ lvs = get_lvs(fields=fields, filters=filters, tags=tags) return lvs[0] if len(lvs) > 0 else [] + + +def get_lv_by_name(name): + stdout, stderr, returncode = process.call( + ['lvs', '--noheadings', '-o', LV_FIELDS, '-S', + 'lv_name={}'.format(name)], + verbose_on_failure=False + ) + lvs = _output_parser(stdout, LV_FIELDS) + return [Volume(**lv) for lv in lvs] + + +def get_lvs_by_tag(lv_tag): + stdout, stderr, returncode = process.call( + ['lvs', '--noheadings', '--separator=";"', '-a', '-o', LV_FIELDS, '-S', + 'lv_tags={{{}}}'.format(lv_tag)], + verbose_on_failure=False + ) + lvs = _output_parser(stdout, LV_FIELDS) + return [Volume(**lv) for lv in lvs] + + +def get_device_lvs(device, name_prefix=''): + stdout, stderr, returncode = process.call( + ['pvs'] + LV_CMD_OPTIONS + ['-o', LV_FIELDS, device], + verbose_on_failure=False + ) + lvs = _output_parser(stdout, LV_FIELDS) + return [Volume(**lv) for lv in lvs if lv['lv_name'] and + lv['lv_name'].startswith(name_prefix)] diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py index 61d227533..e4ac074a4 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -15,30 +15,40 @@ from .listing import direct_report logger = logging.getLogger(__name__) -def activate_filestore(lvs, no_systemd=False): +def activate_filestore(osd_lvs, no_systemd=False): # find the osd - osd_lv = lvs.get(lv_tags={'ceph.type': 'data'}) - if not osd_lv: + for osd_lv in osd_lvs: + if osd_lv.tags.get('ceph.type') == 'data': + data_lv = osd_lv + break + else: raise RuntimeError('Unable to find a data LV for filestore activation') - is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' - is_vdo = osd_lv.tags.get('ceph.vdo', '0') - osd_id = osd_lv.tags['ceph.osd_id'] - configuration.load_ceph_conf_path(osd_lv.tags['ceph.cluster_name']) + is_encrypted = data_lv.tags.get('ceph.encrypted', '0') == '1' + is_vdo = data_lv.tags.get('ceph.vdo', '0') + + osd_id = data_lv.tags['ceph.osd_id'] + configuration.load_ceph_conf_path(data_lv.tags['ceph.cluster_name']) configuration.load() # it may have a volume with a journal - osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'}) + for osd_lv in osd_lvs: + if osd_lv.tags.get('ceph.type') == 'journal': + osd_journal_lv = osd_lv + break + else: + osd_journal_lv = None + # TODO: add sensible error reporting if this is ever the case # blow up with a KeyError if this doesn't exist - osd_fsid = osd_lv.tags['ceph.osd_fsid'] + osd_fsid = data_lv.tags['ceph.osd_fsid'] if not osd_journal_lv: # must be a disk partition, by querying blkid by the uuid we are ensuring that the # device path is always correct - journal_uuid = osd_lv.tags['ceph.journal_uuid'] + journal_uuid = data_lv.tags['ceph.journal_uuid'] osd_journal = disk.get_device_from_partuuid(journal_uuid) else: journal_uuid = osd_journal_lv.lv_uuid - osd_journal = osd_lv.tags['ceph.journal_device'] + osd_journal = data_lv.tags['ceph.journal_device'] if not osd_journal: raise RuntimeError('unable to detect an lv or device journal for OSD %s' % osd_id) @@ -46,17 +56,17 @@ def activate_filestore(lvs, no_systemd=False): # this is done here, so that previous checks that ensure path availability # and correctness can still be enforced, and report if any issues are found if is_encrypted: - lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret'] + lockbox_secret = data_lv.tags['ceph.cephx_lockbox_secret'] # this keyring writing is idempotent encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret) dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid) - encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid) + encryption_utils.luks_open(dmcrypt_secret, data_lv.lv_path, data_lv.lv_uuid) encryption_utils.luks_open(dmcrypt_secret, osd_journal, journal_uuid) osd_journal = '/dev/mapper/%s' % journal_uuid - source = '/dev/mapper/%s' % osd_lv.lv_uuid + source = '/dev/mapper/%s' % data_lv.lv_uuid else: - source = osd_lv.lv_path + source = data_lv.lv_path # mount the osd destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) @@ -86,24 +96,33 @@ def activate_filestore(lvs, no_systemd=False): terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id) -def get_osd_device_path(osd_lv, lvs, device_type, dmcrypt_secret=None): +def get_osd_device_path(osd_lvs, device_type, dmcrypt_secret=None): """ - ``device_type`` can be one of ``db``, ``wal`` or ``block`` so that - we can query ``lvs`` (a ``Volumes`` object) and fallback to querying the uuid - if that is not present. + ``device_type`` can be one of ``db``, ``wal`` or ``block`` so that we can + query LVs on system and fallback to querying the uuid if that is not + present. - Return a path if possible, failing to do that a ``None``, since some of these devices - are optional + Return a path if possible, failing to do that a ``None``, since some of + these devices are optional. """ - osd_lv = lvs.get(lv_tags={'ceph.type': 'block'}) - is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' - logger.debug('Found block device (%s) with encryption: %s', osd_lv.name, is_encrypted) - uuid_tag = 'ceph.%s_uuid' % device_type - device_uuid = osd_lv.tags.get(uuid_tag) - if not device_uuid: - return None - - device_lv = lvs.get(lv_tags={'ceph.type': device_type}) + osd_block_lv = None + for lv in osd_lvs: + if lv.tags.get('ceph.type') == 'block': + osd_block_lv = lv + break + if osd_block_lv: + is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1' + logger.debug('Found block device (%s) with encryption: %s', osd_block_lv.name, is_encrypted) + uuid_tag = 'ceph.%s_uuid' % device_type + device_uuid = osd_block_lv.tags.get(uuid_tag) + if not device_uuid: + return None + + device_lv = None + for lv in osd_lvs: + if lv.tags.get('ceph.type') == device_type: + device_lv = lv + break if device_lv: if is_encrypted: encryption_utils.luks_open(dmcrypt_secret, device_lv.lv_path, device_uuid) @@ -121,16 +140,19 @@ def get_osd_device_path(osd_lv, lvs, device_type, dmcrypt_secret=None): raise RuntimeError('could not find %s with uuid %s' % (device_type, device_uuid)) -def activate_bluestore(lvs, no_systemd=False): - # find the osd - osd_lv = lvs.get(lv_tags={'ceph.type': 'block'}) - if not osd_lv: +def activate_bluestore(osd_lvs, no_systemd=False): + for lv in osd_lvs: + if lv.tags.get('ceph.type') == 'block': + osd_block_lv = lv + break + else: raise RuntimeError('could not find a bluestore OSD to activate') - is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' + + is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1' dmcrypt_secret = None - osd_id = osd_lv.tags['ceph.osd_id'] - conf.cluster = osd_lv.tags['ceph.cluster_name'] - osd_fsid = osd_lv.tags['ceph.osd_fsid'] + osd_id = osd_block_lv.tags['ceph.osd_id'] + conf.cluster = osd_block_lv.tags['ceph.cluster_name'] + osd_fsid = osd_block_lv.tags['ceph.osd_fsid'] # mount on tmpfs the osd directory osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) @@ -145,16 +167,16 @@ def activate_bluestore(lvs, no_systemd=False): os.unlink(os.path.join(osd_path, link_name)) # encryption is handled here, before priming the OSD dir if is_encrypted: - osd_lv_path = '/dev/mapper/%s' % osd_lv.lv_uuid - lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret'] + osd_lv_path = '/dev/mapper/%s' % osd_block_lv.lv_uuid + lockbox_secret = osd_block_lv.tags['ceph.cephx_lockbox_secret'] encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret) dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid) - encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid) + encryption_utils.luks_open(dmcrypt_secret, osd_block_lv.lv_path, osd_block_lv.lv_uuid) else: - osd_lv_path = osd_lv.lv_path + osd_lv_path = osd_block_lv.lv_path - db_device_path = get_osd_device_path(osd_lv, lvs, 'db', dmcrypt_secret=dmcrypt_secret) - wal_device_path = get_osd_device_path(osd_lv, lvs, 'wal', dmcrypt_secret=dmcrypt_secret) + db_device_path = get_osd_device_path(osd_lvs, 'db', dmcrypt_secret=dmcrypt_secret) + wal_device_path = get_osd_device_path(osd_lvs, 'wal', dmcrypt_secret=dmcrypt_secret) # Once symlinks are removed, the osd dir can be 'primed again. chown first, # regardless of what currently exists so that ``prime-osd-dir`` can succeed @@ -235,36 +257,43 @@ class Activate(object): def activate(self, args, osd_id=None, osd_fsid=None): """ :param args: The parsed arguments coming from the CLI - :param osd_id: When activating all, this gets populated with an existing OSD ID - :param osd_fsid: When activating all, this gets populated with an existing OSD FSID + :param osd_id: When activating all, this gets populated with an + existing OSD ID + :param osd_fsid: When activating all, this gets populated with an + existing OSD FSID """ - osd_id = osd_id if osd_id is not None else args.osd_id - osd_fsid = osd_fsid if osd_fsid is not None else args.osd_fsid + osd_id = osd_id if osd_id else args.osd_id + osd_fsid = osd_fsid if osd_fsid else args.osd_fsid - lvs = api.Volumes() - # filter them down for the OSD ID and FSID we need to activate if osd_id and osd_fsid: - lvs.filter(lv_tags={'ceph.osd_id': osd_id, 'ceph.osd_fsid': osd_fsid}) - elif osd_fsid and not osd_id: - lvs.filter(lv_tags={'ceph.osd_fsid': osd_fsid}) + tags = {'ceph.osd_id': osd_id, 'ceph.osd_fsid': osd_fsid} + elif not osd_id and osd_fsid: + tags = {'ceph.osd_fsid': osd_fsid} + lvs = api.get_lvs(tags=tags) if not lvs: - raise RuntimeError('could not find osd.%s with osd_fsid %s' % (osd_id, osd_fsid)) + raise RuntimeError('could not find osd.%s with osd_fsid %s' % + (osd_id, osd_fsid)) + # This argument is only available when passed in directly or via # systemd, not when ``create`` is being used if getattr(args, 'auto_detect_objectstore', False): logger.info('auto detecting objectstore') - # may get multiple lvs, so can't do lvs.get() calls here + # may get multiple lvs, so can't do get_the_lvs() calls here for lv in lvs: has_journal = lv.tags.get('ceph.journal_uuid') if has_journal: - logger.info('found a journal associated with the OSD, assuming filestore') - return activate_filestore(lvs, no_systemd=args.no_systemd) - logger.info('unable to find a journal associated with the OSD, assuming bluestore') - return activate_bluestore(lvs, no_systemd=args.no_systemd) + logger.info('found a journal associated with the OSD, ' + 'assuming filestore') + return activate_filestore(lvs, args.no_systemd) + + logger.info('unable to find a journal associated with the OSD, ' + 'assuming bluestore') + + return activate_bluestore(lvs, args.no_systemd) if args.bluestore: - activate_bluestore(lvs, no_systemd=args.no_systemd) + activate_bluestore(lvs, args.no_systemd) elif args.filestore: - activate_filestore(lvs, no_systemd=args.no_systemd) + activate_filestore(lvs, args.no_systemd) def main(self): sub_command_help = dedent(""" diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py index b7a4b35b2..a83c8680b 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py @@ -1,5 +1,6 @@ import argparse import logging +import json from textwrap import dedent from ceph_volume import terminal, decorators from ceph_volume.util import disk, prompt_bool @@ -376,6 +377,11 @@ class Batch(object): # filtered if self.args.yes and dev_list and self.usable and devs != usable: err = '{} devices were filtered in non-interactive mode, bailing out' + if self.args.format == "json" and self.args.report: + # if a json report is requested, report unchanged so idempotency checks + # in ceph-ansible will work + print(json.dumps({"changed": False, "osds": [], "vgs": []})) + raise SystemExit(0) raise RuntimeError(err.format(len(devs) - len(usable))) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py index 4db853280..26ea2bf21 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py @@ -135,21 +135,6 @@ class Prepare(object): raise RuntimeError('unable to use device') return uuid - def get_lv(self, argument): - """ - Perform some parsing of the command-line value so that the process - can determine correctly if it got a device path or an lv. - - :param argument: The command-line value that will need to be split to - retrieve the actual lv - """ - #TODO is this efficient? - try: - vg_name, lv_name = argument.split('/') - except (ValueError, AttributeError): - return None - return api.get_lv(lv_name=lv_name, vg_name=vg_name) - def setup_device(self, device_type, device_name, tags, size): """ Check if ``device`` is an lv, if so, set the tags, making sure to @@ -163,7 +148,14 @@ class Prepare(object): return '', '', tags tags['ceph.type'] = device_type tags['ceph.vdo'] = api.is_vdo(device_name) - lv = self.get_lv(device_name) + + try: + vg_name, lv_name = device_name.split('/') + lv = api.get_first_lv(filters={'lv_name': lv_name, + 'vg_name': vg_name}) + except ValueError: + lv = None + if lv: uuid = lv.lv_uuid path = lv.lv_path @@ -239,7 +231,15 @@ class Prepare(object): """ if args is not None: self.args = args - if api.is_ceph_device(self.get_lv(self.args.data)): + + try: + vgname, lvname = self.args.data.split('/') + lv = api.get_first_lv(filters={'lv_name': lvname, + 'vg_name': vgname}) + except ValueError: + lv = None + + if api.is_ceph_device(lv): logger.info("device {} is already used".format(self.args.data)) raise RuntimeError("skipping {}, it is already prepared".format(self.args.data)) try: @@ -298,7 +298,13 @@ class Prepare(object): if not self.args.journal: raise RuntimeError('--journal is required when using --filestore') - data_lv = self.get_lv(self.args.data) + try: + vg_name, lv_name = self.args.data.split('/') + data_lv = api.get_first_lv(filters={'lv_name': lv_name, + 'vg_name': vg_name}) + except ValueError: + data_lv = None + if not data_lv: data_lv = self.prepare_data_device('data', osd_fsid) @@ -323,7 +329,13 @@ class Prepare(object): osd_fsid, ) elif self.args.bluestore: - block_lv = self.get_lv(self.args.data) + try: + vg_name, lv_name = self.args.data.split('/') + block_lv = api.get_first_lv(filters={'lv_name': lv_name, + 'vg_name': vg_name}) + except ValueError: + block_lv = None + if not block_lv: block_lv = self.prepare_data_device('block', osd_fsid) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/bluestore.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/bluestore.py index aa5f1a94f..cdaabbeec 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/bluestore.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/bluestore.py @@ -130,7 +130,6 @@ class MixedType(MixedStrategy): super(MixedType, self).__init__(args, data_devs, db_devs, wal_devs) self.block_db_size = self.get_block_db_size() self.block_wal_size = self.get_block_wal_size() - self.system_vgs = lvm.VolumeGroups() self.common_vg = None self.common_wal_vg = None self.dbs_needed = len(self.data_devs) * self.osds_per_device diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/filestore.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/filestore.py index 0c9ff31fb..bc10473ee 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/filestore.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/strategies/filestore.py @@ -184,7 +184,6 @@ class MixedType(MixedStrategy): self.blank_journal_devs = [] self.journals_needed = len(self.data_devs) * self.osds_per_device self.journal_size = get_journal_size(args) - self.system_vgs = lvm.VolumeGroups() self.validate_compute() @classmethod diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py index ec579a145..63624e55f 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py @@ -7,7 +7,7 @@ from textwrap import dedent from ceph_volume import decorators, terminal, process from ceph_volume.api import lvm as api -from ceph_volume.util import system, encryption, disk, arg_validators, str_to_int +from ceph_volume.util import system, encryption, disk, arg_validators, str_to_int, merge_dict from ceph_volume.util.device import Device from ceph_volume.systemd import systemctl @@ -81,17 +81,17 @@ def find_associated_devices(osd_id=None, osd_fsid=None): lv_tags['ceph.osd_id'] = osd_id if osd_fsid: lv_tags['ceph.osd_fsid'] = osd_fsid - lvs = api.Volumes() - lvs.filter(lv_tags=lv_tags) - if not lvs: - raise RuntimeError('Unable to find any LV for zapping OSD: %s' % osd_id or osd_fsid) - devices_to_zap = ensure_associated_lvs(lvs) + lvs = api.get_lvs(tags=lv_tags) + if not lvs: + raise RuntimeError('Unable to find any LV for zapping OSD: ' + '%s' % osd_id or osd_fsid) + devices_to_zap = ensure_associated_lvs(lvs, lv_tags) return [Device(path) for path in set(devices_to_zap) if path] -def ensure_associated_lvs(lvs): +def ensure_associated_lvs(lvs, lv_tags={}): """ Go through each LV and ensure if backing devices (journal, wal, block) are LVs or partitions, so that they can be accurately reported. @@ -100,14 +100,12 @@ def ensure_associated_lvs(lvs): # receive a filtering for osd.1, and have multiple failed deployments # leaving many journals with osd.1 - usually, only a single LV will be # returned - journal_lvs = lvs._filter(lv_tags={'ceph.type': 'journal'}) - db_lvs = lvs._filter(lv_tags={'ceph.type': 'db'}) - wal_lvs = lvs._filter(lv_tags={'ceph.type': 'wal'}) - backing_devices = [ - (journal_lvs, 'journal'), - (db_lvs, 'db'), - (wal_lvs, 'wal') - ] + + journal_lvs = api.get_lvs(tags=merge_dict(lv_tags, {'ceph.type': 'journal'})) + db_lvs = api.get_lvs(tags=merge_dict(lv_tags, {'ceph.type': 'db'})) + wal_lvs = api.get_lvs(tags=merge_dict(lv_tags, {'ceph.type': 'wal'})) + backing_devices = [(journal_lvs, 'journal'), (db_lvs, 'db'), + (wal_lvs, 'wal')] verified_devices = [] @@ -168,21 +166,27 @@ class Zap(object): Device examples: vg-name/lv-name, /dev/vg-name/lv-name Requirements: Must be a logical volume (LV) """ - lv = api.get_lv(lv_name=device.lv_name, vg_name=device.vg_name) + lv = api.get_first_lv(filters={'lv_name': device.lv_name, 'vg_name': + device.vg_name}) self.unmount_lv(lv) wipefs(device.abspath) zap_data(device.abspath) if self.args.destroy: - lvs = api.Volumes() - lvs.filter(vg_name=device.vg_name) - if len(lvs) <= 1: - mlogger.info('Only 1 LV left in VG, will proceed to destroy volume group %s', device.vg_name) + lvs = api.get_lvs(filters={'vg_name': device.vg_name}) + if lvs == []: + mlogger.info('No LVs left, exiting', device.vg_name) + return + elif len(lvs) <= 1: + mlogger.info('Only 1 LV left in VG, will proceed to destroy ' + 'volume group %s', device.vg_name) api.remove_vg(device.vg_name) else: - mlogger.info('More than 1 LV left in VG, will proceed to destroy LV only') - mlogger.info('Removing LV because --destroy was given: %s', device.abspath) + mlogger.info('More than 1 LV left in VG, will proceed to ' + 'destroy LV only') + mlogger.info('Removing LV because --destroy was given: %s', + device.abspath) api.remove_lv(device.abspath) elif lv: # just remove all lvm metadata, leaving the LV around diff --git a/ceph/src/ceph-volume/ceph_volume/devices/raw/common.py b/ceph/src/ceph-volume/ceph_volume/devices/raw/common.py index d34a2941d..08cfd0289 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/raw/common.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/raw/common.py @@ -26,10 +26,6 @@ def create_parser(prog, description): dest='crush_device_class', help='Crush device class to assign this OSD to', ) - parser.add_argument( - '--cluster-fsid', - help='Specify the cluster fsid, useful when no ceph.conf is available', - ) parser.add_argument( '--no-tmpfs', action='store_true', @@ -45,4 +41,9 @@ def create_parser(prog, description): dest='block_wal', help='Path to bluestore block.wal block device' ) + parser.add_argument( + '--dmcrypt', + action='store_true', + help='Enable device encryption via dm-crypt', + ) return parser diff --git a/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py b/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py index b04f55cd8..bb15bf199 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py @@ -30,8 +30,34 @@ class List(object): if not devs: logger.debug('Listing block devices via lsblk...') devs = [] + # adding '--inverse' allows us to get the mapper devices list in that command output. + # not listing root devices containing partitions shouldn't have side effect since we are + # in `ceph-volume raw` context. + # + # example: + # running `lsblk --paths --nodeps --output=NAME --noheadings` doesn't allow to get the mapper list + # because the output is like following : + # + # $ lsblk --paths --nodeps --output=NAME --noheadings + # /dev/sda + # /dev/sdb + # /dev/sdc + # /dev/sdd + # + # the dmcrypt mappers are hidden because of the `--nodeps` given they are displayed as a dependency. + # + # $ lsblk --paths --output=NAME --noheadings + # /dev/sda + # |-/dev/mapper/ceph-3b52c90d-6548-407d-bde1-efd31809702f-sda-block-dmcrypt + # `-/dev/mapper/ceph-3b52c90d-6548-407d-bde1-efd31809702f-sda-db-dmcrypt + # /dev/sdb + # /dev/sdc + # /dev/sdd + # + # adding `--inverse` is a trick to get around this issue, the counterpart is that we can't list root devices if they contain + # at least one partition but this shouldn't be an issue in `ceph-volume raw` context given we only deal with raw devices. out, err, ret = process.call([ - 'lsblk', '--paths', '--nodeps', '--output=NAME', '--noheadings' + 'lsblk', '--paths', '--nodeps', '--output=NAME', '--noheadings', '--inverse' ]) assert not ret devs = out diff --git a/ceph/src/ceph-volume/ceph_volume/devices/raw/prepare.py b/ceph/src/ceph-volume/ceph_volume/devices/raw/prepare.py index cb5c59ce4..3c96eedac 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/raw/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/raw/prepare.py @@ -1,15 +1,39 @@ from __future__ import print_function import json import logging +import os from textwrap import dedent from ceph_volume.util import prepare as prepare_utils +from ceph_volume.util import encryption as encryption_utils +from ceph_volume.util import disk from ceph_volume.util import system -from ceph_volume import conf, decorators, terminal +from ceph_volume import decorators, terminal from ceph_volume.devices.lvm.common import rollback_osd from .common import create_parser logger = logging.getLogger(__name__) +def prepare_dmcrypt(key, device, device_type, fsid): + """ + Helper for devices that are encrypted. The operations needed for + block, db, wal, or data/journal devices are all the same + """ + if not device: + return '' + kname = disk.lsblk(device)['KNAME'] + mapping = 'ceph-{}-{}-{}-dmcrypt'.format(fsid, kname, device_type) + # format data device + encryption_utils.luks_format( + key, + device + ) + encryption_utils.luks_open( + key, + device, + mapping + ) + + return '/dev/mapper/{}'.format(mapping) def prepare_bluestore(block, wal, db, secrets, osd_id, fsid, tmpfs): """ @@ -22,6 +46,12 @@ def prepare_bluestore(block, wal, db, secrets, osd_id, fsid, tmpfs): """ cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) + if secrets.get('dmcrypt_key'): + key = secrets['dmcrypt_key'] + block = prepare_dmcrypt(key, block, 'block', fsid) + wal = prepare_dmcrypt(key, wal, 'wal', fsid) + db = prepare_dmcrypt(key, db, 'db', fsid) + # create the directory prepare_utils.create_osd_path(osd_id, tmpfs=tmpfs) # symlink the block @@ -64,21 +94,20 @@ class Prepare(object): logger.info('will rollback OSD ID creation') rollback_osd(self.args, self.osd_id) raise - terminal.success("ceph-volume raw prepare successful for: %s" % self.args.data) - - def get_cluster_fsid(self): - """ - Allows using --cluster-fsid as an argument, but can fallback to reading - from ceph.conf if that is unset (the default behavior). - """ - if self.args.cluster_fsid: - return self.args.cluster_fsid + dmcrypt_log = 'dmcrypt' if args.dmcrypt else 'clear' + terminal.success("ceph-volume raw {} prepare successful for: {}".format(dmcrypt_log, self.args.data)) - return conf.ceph.get('global', 'fsid') @decorators.needs_root def prepare(self): secrets = {'cephx_secret': prepare_utils.create_key()} + encrypted = 1 if self.args.dmcrypt else 0 + cephx_lockbox_secret = '' if not encrypted else prepare_utils.create_key() + + if encrypted: + secrets['dmcrypt_key'] = os.getenv('CEPH_VOLUME_DMCRYPT_SECRET') + secrets['cephx_lockbox_secret'] = cephx_lockbox_secret # dummy value to make `ceph osd new` not complaining + osd_fsid = system.generate_uuid() crush_device_class = self.args.crush_device_class if crush_device_class: @@ -94,6 +123,7 @@ class Prepare(object): # reuse a given ID if it exists, otherwise create a new ID self.osd_id = prepare_utils.create_id( osd_fsid, json.dumps(secrets)) + prepare_bluestore( self.args.data, wal, @@ -112,8 +142,6 @@ class Prepare(object): Once the OSD is ready, an ad-hoc systemd unit will be enabled so that it can later get activated and the OSD daemon can get started. - Encryption is not supported. - ceph-volume raw prepare --bluestore --data {device} DB and WAL devices are supported. @@ -132,5 +160,10 @@ class Prepare(object): if not self.args.bluestore: terminal.error('must specify --bluestore (currently the only supported backend)') raise SystemExit(1) + if self.args.dmcrypt and not os.getenv('CEPH_VOLUME_DMCRYPT_SECRET'): + terminal.error('encryption was requested (--dmcrypt) but environment variable ' \ + 'CEPH_VOLUME_DMCRYPT_SECRET is not set, you must set ' \ + 'this variable to provide a dmcrypt secret.') + raise SystemExit(1) self.safe_prepare(self.args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py b/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py index 1e3deae4c..0f83b37ef 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py @@ -80,7 +80,7 @@ class Scan(object): device = os.readlink(path) else: device = path - lvm_device = lvm.get_lv_from_argument(device) + lvm_device = lvm.get_first_lv(filters={'lv_path': device}) if lvm_device: device_uuid = lvm_device.lv_uuid else: diff --git a/ceph/src/ceph-volume/ceph_volume/drive_group/__init__.py b/ceph/src/ceph-volume/ceph_volume/drive_group/__init__.py new file mode 100644 index 000000000..14a0fd721 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/drive_group/__init__.py @@ -0,0 +1 @@ +from .main import Deploy # noqa diff --git a/ceph/src/ceph-volume/ceph_volume/drive_group/main.py b/ceph/src/ceph-volume/ceph_volume/drive_group/main.py new file mode 100644 index 000000000..684e89f36 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/drive_group/main.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +import argparse +import json +import logging +import sys + +from ceph.deployment.drive_group import DriveGroupSpec +from ceph.deployment.drive_selection.selector import DriveSelection +from ceph.deployment.translate import to_ceph_volume +from ceph.deployment.inventory import Device +from ceph_volume.inventory import Inventory +from ceph_volume.devices.lvm.batch import Batch + +logger = logging.getLogger(__name__) + +class Deploy(object): + + help = ''' + Deploy OSDs according to a drive groups specification. + + The DriveGroup specification must be passed in json. + It can either be (preference in this order) + - in a file, path passed as a positional argument + - read from stdin, pass "-" as a positional argument + - a json string passed via the --spec argument + + Either the path postional argument or --spec must be specifed. + ''' + + def __init__(self, argv): + logger.error(f'argv: {argv}') + self.argv = argv + + def main(self): + parser = argparse.ArgumentParser( + prog='ceph-volume drive-group', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=self.help, + ) + parser.add_argument( + 'path', + nargs='?', + default=None, + help=('Path to file containing drive group spec or "-" to read from stdin'), + ) + parser.add_argument( + '--spec', + default='', + nargs='?', + help=('drive-group json string') + ) + parser.add_argument( + '--dry-run', + default=False, + action='store_true', + help=('dry run, only print the batch command that would be run'), + ) + self.args = parser.parse_args(self.argv) + if self.args.path: + if self.args.path == "-": + commands = self.from_json(sys.stdin) + else: + with open(self.args.path, 'r') as f: + commands = self.from_json(f) + elif self.args.spec: + dg = json.loads(self.args.spec) + commands = self.get_dg_spec(dg) + else: + # either --spec or path arg must be specified + parser.print_help(sys.stderr) + sys.exit(0) + cmd = commands.run() + if not cmd: + logger.error('DriveGroup didn\'t produce any commands') + return + if self.args.dry_run: + logger.info('Returning ceph-volume command (--dry-run was passed): {}'.format(cmd)) + print(cmd) + else: + logger.info('Running ceph-volume command: {}'.format(cmd)) + batch_args = cmd.split(' ')[2:] + b = Batch(batch_args) + b.main() + + def from_json(self, file_): + dg = {} + dg = json.load(file_) + return self.get_dg_spec(dg) + + def get_dg_spec(self, dg): + dg_spec = DriveGroupSpec._from_json_impl(dg) + dg_spec.validate() + i = Inventory([]) + i.main() + inventory = i.get_report() + devices = [Device.from_json(i) for i in inventory] + selection = DriveSelection(dg_spec, devices) + return to_ceph_volume(selection) diff --git a/ceph/src/ceph-volume/ceph_volume/exceptions.py b/ceph/src/ceph-volume/ceph_volume/exceptions.py index f40b7b11d..5c6429483 100644 --- a/ceph/src/ceph-volume/ceph_volume/exceptions.py +++ b/ceph/src/ceph-volume/ceph_volume/exceptions.py @@ -50,37 +50,6 @@ class SuperUserError(Exception): return 'This command needs to be executed with sudo or as root' -class MultiplePVsError(Exception): - - def __init__(self, pv_name): - self.pv_name = pv_name - - def __str__(self): - msg = "Got more than 1 result looking for physical volume: %s" % self.pv_name - return msg - - -class MultipleLVsError(Exception): - - def __init__(self, lv_name, lv_path): - self.lv_name = lv_name - self.lv_path = lv_path - - def __str__(self): - msg = "Got more than 1 result looking for %s with path: %s" % (self.lv_name, self.lv_path) - return msg - - -class MultipleVGsError(Exception): - - def __init__(self, vg_name): - self.vg_name = vg_name - - def __str__(self): - msg = "Got more than 1 result looking for volume group: %s" % self.vg_name - return msg - - class SizeAllocationError(Exception): def __init__(self, requested, available): diff --git a/ceph/src/ceph-volume/ceph_volume/inventory/main.py b/ceph/src/ceph-volume/ceph_volume/inventory/main.py index 1d821b602..470be274f 100644 --- a/ceph/src/ceph-volume/ceph_volume/inventory/main.py +++ b/ceph/src/ceph-volume/ceph_volume/inventory/main.py @@ -37,6 +37,12 @@ class Inventory(object): else: self.format_report(Devices()) + def get_report(self): + if self.args.path: + return Device(self.args.path).json_report() + else: + return Devices().json_report() + def format_report(self, inventory): if self.args.format == 'json': print(json.dumps(inventory.json_report())) diff --git a/ceph/src/ceph-volume/ceph_volume/main.py b/ceph/src/ceph-volume/ceph_volume/main.py index 8c5801c5d..461395ae5 100644 --- a/ceph/src/ceph-volume/ceph_volume/main.py +++ b/ceph/src/ceph-volume/ceph_volume/main.py @@ -6,7 +6,7 @@ import sys import logging from ceph_volume.decorators import catches -from ceph_volume import log, devices, configuration, conf, exceptions, terminal, inventory +from ceph_volume import log, devices, configuration, conf, exceptions, terminal, inventory, drive_group class Volume(object): @@ -29,6 +29,7 @@ Ceph Conf: {ceph_path} 'simple': devices.simple.Simple, 'raw': devices.raw.Raw, 'inventory': inventory.Inventory, + 'drive-group': drive_group.Deploy, } self.plugin_help = "No plugins found/loaded" if argv is None: diff --git a/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py b/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py index d75920bd6..fe2fe8b57 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py @@ -31,188 +31,6 @@ class TestParseTags(object): assert result['ceph.fsid'] == '0000' -class TestGetAPIVgs(object): - - def test_report_is_emtpy(self, monkeypatch): - monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('\n\n', '', 0)) - assert api.get_api_vgs() == [] - - def test_report_has_stuff(self, monkeypatch): - report = [' VolGroup00'] - monkeypatch.setattr(api.process, 'call', lambda x, **kw: (report, '', 0)) - assert api.get_api_vgs() == [{'vg_name': 'VolGroup00'}] - - def test_report_has_stuff_with_empty_attrs(self, monkeypatch): - report = [' VolGroup00 ;;;;;;4194304'] - monkeypatch.setattr(api.process, 'call', lambda x, **kw: (report, '', 0)) - result = api.get_api_vgs()[0] - assert len(result.keys()) == 7 - assert result['vg_name'] == 'VolGroup00' - assert result['vg_extent_size'] == '4194304' - - def test_report_has_multiple_items(self, monkeypatch): - report = [' VolGroup00;;;;;;;', ' ceph_vg;;;;;;;'] - monkeypatch.setattr(api.process, 'call', lambda x, **kw: (report, '', 0)) - result = api.get_api_vgs() - assert result[0]['vg_name'] == 'VolGroup00' - assert result[1]['vg_name'] == 'ceph_vg' - - -class TestGetAPILvs(object): - - def test_report_is_emtpy(self, monkeypatch): - monkeypatch.setattr(api.process, 'call', lambda x, **kw: ('', '', 0)) - assert api.get_api_lvs() == [] - - def test_report_has_stuff(self, monkeypatch): - report = [' ;/path;VolGroup00;root'] - monkeypatch.setattr(api.process, 'call', lambda x, **kw: (report, '', 0)) - result = api.get_api_lvs() - assert result[0]['lv_name'] == 'VolGroup00' - - def test_report_has_multiple_items(self, monkeypatch): - report = [' ;/path;VolName;root', ';/dev/path;ceph_lv;ceph_vg'] - monkeypatch.setattr(api.process, 'call', lambda x, **kw: (report, '', 0)) - result = api.get_api_lvs() - assert result[0]['lv_name'] == 'VolName' - assert result[1]['lv_name'] == 'ceph_lv' - - -@pytest.fixture -def volumes(monkeypatch): - monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) - volumes = api.Volumes() - volumes._purge() - # also patch api.Volumes so that when it is called, it will use the newly - # created fixture, with whatever the test method wants to append to it - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - return volumes - - -@pytest.fixture -def volume_groups(monkeypatch): - monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) - vgs = api.VolumeGroups() - vgs._purge() - return vgs - - -class TestGetLV(object): - - def test_nothing_is_passed_in(self): - # so we return a None - assert api.get_lv() is None - - def test_single_lv_is_matched(self, volumes, monkeypatch): - FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', lv_tags="ceph.type=data") - volumes.append(FooVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - assert api.get_lv(lv_name='foo') == FooVolume - - def test_single_lv_is_matched_by_uuid(self, volumes, monkeypatch): - FooVolume = api.Volume( - lv_name='foo', lv_path='/dev/vg/foo', - lv_uuid='1111', lv_tags="ceph.type=data") - volumes.append(FooVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - assert api.get_lv(lv_uuid='1111') == FooVolume - - -class TestGetPV(object): - - def test_nothing_is_passed_in(self): - # so we return a None - assert api.get_pv() is None - - def test_single_pv_is_not_matched(self, pvolumes, monkeypatch): - FooPVolume = api.PVolume(pv_name='/dev/sda', pv_uuid="0000", pv_tags={}, vg_name="vg") - pvolumes.append(FooPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda: pvolumes) - assert api.get_pv(pv_uuid='foo') is None - - def test_single_pv_is_matched(self, pvolumes, monkeypatch): - FooPVolume = api.PVolume(vg_name="vg", pv_name='/dev/sda', pv_uuid="0000", pv_tags={}) - pvolumes.append(FooPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda: pvolumes) - assert api.get_pv(pv_uuid='0000') == FooPVolume - - def test_multiple_pvs_is_matched_by_uuid(self, pvolumes, monkeypatch): - FooPVolume = api.PVolume(vg_name="vg", pv_name='/dev/sda', pv_uuid="0000", pv_tags={}, lv_uuid="0000000") - BarPVolume = api.PVolume(vg_name="vg", pv_name='/dev/sda', pv_uuid="0000", pv_tags={}) - pvolumes.append(FooPVolume) - pvolumes.append(BarPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda: pvolumes) - assert api.get_pv(pv_uuid='0000') == FooPVolume - - def test_multiple_pvs_is_matched_by_name(self, pvolumes, monkeypatch): - FooPVolume = api.PVolume(vg_name="vg", pv_name='/dev/sda', pv_uuid="0000", pv_tags={}, lv_uuid="0000000") - BarPVolume = api.PVolume(vg_name="vg", pv_name='/dev/sda', pv_uuid="0000", pv_tags={}) - pvolumes.append(FooPVolume) - pvolumes.append(BarPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda: pvolumes) - assert api.get_pv(pv_name='/dev/sda') == FooPVolume - - def test_multiple_pvs_is_matched_by_tags(self, pvolumes, monkeypatch): - FooPVolume = api.PVolume(vg_name="vg1", pv_name='/dev/sdc', pv_uuid="1000", pv_tags="ceph.foo=bar", lv_uuid="0000000") - BarPVolume = api.PVolume(vg_name="vg", pv_name='/dev/sda', pv_uuid="0000", pv_tags="ceph.foo=bar") - pvolumes.append(FooPVolume) - pvolumes.append(BarPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda: pvolumes) - with pytest.raises(exceptions.MultiplePVsError): - api.get_pv(pv_tags={"ceph.foo": "bar"}) - - def test_single_pv_is_matched_by_uuid(self, pvolumes, monkeypatch): - FooPVolume = api.PVolume( - pv_name='/dev/vg/foo', - pv_uuid='1111', pv_tags="ceph.type=data", vg_name="vg") - pvolumes.append(FooPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda: pvolumes) - assert api.get_pv(pv_uuid='1111') == FooPVolume - - def test_vg_name_is_set(self, pvolumes, monkeypatch): - FooPVolume = api.PVolume( - pv_name='/dev/vg/foo', - pv_uuid='1111', pv_tags="ceph.type=data", vg_name="vg") - pvolumes.append(FooPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda: pvolumes) - pv = api.get_pv(pv_name="/dev/vg/foo") - assert pv.vg_name == "vg" - - -class TestPVolumes(object): - - def test_filter_by_tag_does_not_match_one(self, pvolumes, monkeypatch): - pv_tags = "ceph.type=journal,ceph.osd_id=1,ceph.fsid=000-aaa" - FooPVolume = api.PVolume( - pv_name='/dev/vg/foo', - pv_uuid='1111', pv_tags=pv_tags, vg_name='vg') - pvolumes.append(FooPVolume) - assert pvolumes.filter(pv_tags={'ceph.type': 'journal', - 'ceph.osd_id': '2'}) == [] - - def test_filter_by_tags_matches(self, pvolumes, monkeypatch): - pv_tags = "ceph.type=journal,ceph.osd_id=1" - FooPVolume = api.PVolume( - pv_name='/dev/vg/foo', - pv_uuid='1111', pv_tags=pv_tags, vg_name="vg") - pvolumes.append(FooPVolume) - assert pvolumes.filter(pv_tags={'ceph.type': 'journal', - 'ceph.osd_id': '1'}) == [FooPVolume] - - -class TestGetVG(object): - - def test_nothing_is_passed_in(self): - # so we return a None - assert api.get_vg() is None - - def test_single_vg_is_matched(self, volume_groups, monkeypatch): - FooVG = api.VolumeGroup(vg_name='foo') - volume_groups.append(FooVG) - monkeypatch.setattr(api, 'VolumeGroups', lambda: volume_groups) - assert api.get_vg(vg_name='foo') == FooVG - - class TestVolume(object): def test_is_ceph_device(self): @@ -223,8 +41,8 @@ class TestVolume(object): @pytest.mark.parametrize('dev',[ '/dev/sdb', api.VolumeGroup(vg_name='foo'), - api.Volume(lv_name='vg/no_osd', lv_tags=''), - api.Volume(lv_name='vg/no_osd', lv_tags='ceph.osd_id=null'), + api.Volume(lv_name='vg/no_osd', lv_tags='', lv_path='lv/path'), + api.Volume(lv_name='vg/no_osd', lv_tags='ceph.osd_id=null', lv_path='lv/path'), None, ]) def test_is_not_ceph_device(self, dev): @@ -235,103 +53,6 @@ class TestVolume(object): api.Volume(lv_name='', lv_tags='') -class TestVolumes(object): - - def test_volume_get_has_no_volumes(self, volumes): - assert volumes.get() is None - - def test_volume_get_filtered_has_no_volumes(self, volumes): - assert volumes.get(lv_name='ceph') is None - - def test_volume_has_multiple_matches(self, volumes): - volume1 = volume2 = api.Volume(lv_name='foo', lv_path='/dev/vg/lv', lv_tags='') - volumes.append(volume1) - volumes.append(volume2) - with pytest.raises(exceptions.MultipleLVsError): - volumes.get(lv_name='foo') - - def test_as_dict_infers_type_from_tags(self, volumes): - lv_tags = "ceph.type=data,ceph.fsid=000-aaa" - osd = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags=lv_tags) - volumes.append(osd) - result = volumes.get(lv_tags={'ceph.type': 'data'}).as_dict() - assert result['type'] == 'data' - - def test_as_dict_populates_path_from_lv_api(self, volumes): - lv_tags = "ceph.type=data,ceph.fsid=000-aaa" - osd = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags=lv_tags) - volumes.append(osd) - result = volumes.get(lv_tags={'ceph.type': 'data'}).as_dict() - assert result['path'] == '/dev/vg/lv' - - def test_find_the_correct_one(self, volumes): - volume1 = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags='') - volume2 = api.Volume(lv_name='volume2', lv_path='/dev/vg/lv', lv_tags='') - volumes.append(volume1) - volumes.append(volume2) - assert volumes.get(lv_name='volume1') == volume1 - - def test_filter_by_tag(self, volumes): - lv_tags = "ceph.type=data,ceph.fsid=000-aaa" - osd = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags=lv_tags) - journal = api.Volume(lv_name='volume2', lv_path='/dev/vg/lv', lv_tags='ceph.type=journal') - volumes.append(osd) - volumes.append(journal) - volumes.filter(lv_tags={'ceph.type': 'data'}) - assert len(volumes) == 1 - assert volumes[0].lv_name == 'volume1' - - def test_filter_by_tag_does_not_match_one(self, volumes): - lv_tags = "ceph.type=data,ceph.fsid=000-aaa" - osd = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags=lv_tags) - journal = api.Volume(lv_name='volume2', lv_path='/dev/vg/lv', lv_tags='ceph.osd_id=1,ceph.type=journal') - volumes.append(osd) - volumes.append(journal) - # note the different osd_id! - volumes.filter(lv_tags={'ceph.type': 'data', 'ceph.osd_id': '2'}) - assert volumes == [] - - def test_filter_by_vg_name(self, volumes): - lv_tags = "ceph.type=data,ceph.fsid=000-aaa" - osd = api.Volume(lv_name='volume1', vg_name='ceph_vg', lv_tags=lv_tags) - journal = api.Volume(lv_name='volume2', vg_name='system_vg', lv_tags='ceph.type=journal') - volumes.append(osd) - volumes.append(journal) - volumes.filter(vg_name='ceph_vg') - assert len(volumes) == 1 - assert volumes[0].lv_name == 'volume1' - - def test_filter_by_lv_path(self, volumes): - osd = api.Volume(lv_name='volume1', lv_path='/dev/volume1', lv_tags='') - journal = api.Volume(lv_name='volume2', lv_path='/dev/volume2', lv_tags='') - volumes.append(osd) - volumes.append(journal) - volumes.filter(lv_path='/dev/volume1') - assert len(volumes) == 1 - assert volumes[0].lv_name == 'volume1' - - def test_filter_by_lv_uuid(self, volumes): - osd = api.Volume(lv_name='volume1', lv_path='/dev/volume1', lv_uuid='1111', lv_tags='') - journal = api.Volume(lv_name='volume2', lv_path='/dev/volume2', lv_uuid='', lv_tags='') - volumes.append(osd) - volumes.append(journal) - volumes.filter(lv_uuid='1111') - assert len(volumes) == 1 - assert volumes[0].lv_name == 'volume1' - - def test_filter_by_lv_uuid_nothing_found(self, volumes): - osd = api.Volume(lv_name='volume1', lv_path='/dev/volume1', lv_uuid='1111', lv_tags='') - journal = api.Volume(lv_name='volume2', lv_path='/dev/volume2', lv_uuid='', lv_tags='') - volumes.append(osd) - volumes.append(journal) - volumes.filter(lv_uuid='22222') - assert volumes == [] - - def test_filter_requires_params(self, volumes): - with pytest.raises(TypeError): - volumes.filter() - - class TestVolumeGroup(object): def test_volume_group_no_empty_name(self): @@ -339,60 +60,6 @@ class TestVolumeGroup(object): api.VolumeGroup(vg_name='') -class TestVolumeGroups(object): - - def test_volume_get_has_no_volume_groups(self, volume_groups): - assert volume_groups.get() is None - - def test_volume_get_filtered_has_no_volumes(self, volume_groups): - assert volume_groups.get(vg_name='ceph') is None - - def test_volume_has_multiple_matches(self, volume_groups): - volume1 = volume2 = api.VolumeGroup(vg_name='foo', lv_path='/dev/vg/lv', lv_tags='') - volume_groups.append(volume1) - volume_groups.append(volume2) - with pytest.raises(exceptions.MultipleVGsError): - volume_groups.get(vg_name='foo') - - def test_find_the_correct_one(self, volume_groups): - volume1 = api.VolumeGroup(vg_name='volume1', lv_tags='') - volume2 = api.VolumeGroup(vg_name='volume2', lv_tags='') - volume_groups.append(volume1) - volume_groups.append(volume2) - assert volume_groups.get(vg_name='volume1') == volume1 - - def test_filter_by_tag(self, volume_groups): - vg_tags = "ceph.group=dmcache" - osd = api.VolumeGroup(vg_name='volume1', vg_tags=vg_tags) - journal = api.VolumeGroup(vg_name='volume2', vg_tags='ceph.group=plain') - volume_groups.append(osd) - volume_groups.append(journal) - volume_groups = volume_groups.filter(vg_tags={'ceph.group': 'dmcache'}) - assert len(volume_groups) == 1 - assert volume_groups[0].vg_name == 'volume1' - - def test_filter_by_tag_does_not_match_one(self, volume_groups): - vg_tags = "ceph.group=dmcache,ceph.disk_type=ssd" - osd = api.VolumeGroup(vg_name='volume1', vg_path='/dev/vg/lv', vg_tags=vg_tags) - volume_groups.append(osd) - volume_groups = volume_groups.filter(vg_tags={'ceph.group': 'data', 'ceph.disk_type': 'ssd'}) - assert volume_groups == [] - - def test_filter_by_vg_name(self, volume_groups): - vg_tags = "ceph.type=data,ceph.fsid=000-aaa" - osd = api.VolumeGroup(vg_name='ceph_vg', vg_tags=vg_tags) - journal = api.VolumeGroup(vg_name='volume2', vg_tags='ceph.type=journal') - volume_groups.append(osd) - volume_groups.append(journal) - volume_groups = volume_groups.filter(vg_name='ceph_vg') - assert len(volume_groups) == 1 - assert volume_groups[0].vg_name == 'ceph_vg' - - def test_filter_requires_params(self, volume_groups): - with pytest.raises(TypeError): - volume_groups = volume_groups.filter() - - class TestVolumeGroupFree(object): def test_integer_gets_produced(self): @@ -491,31 +158,6 @@ class TestVolumeGroupSizing(object): self.vg.sizing(size=2048) -class TestGetLVFromArgument(object): - - def setup(self): - self.foo_volume = api.Volume( - lv_name='foo', lv_path='/path/to/lv', - vg_name='foo_group', lv_tags='' - ) - - def test_non_absolute_path_is_not_valid(self, volumes): - volumes.append(self.foo_volume) - assert api.get_lv_from_argument('foo') is None - - def test_too_many_slashes_is_invalid(self, volumes): - volumes.append(self.foo_volume) - assert api.get_lv_from_argument('path/to/lv') is None - - def test_absolute_path_is_not_lv(self, volumes): - volumes.append(self.foo_volume) - assert api.get_lv_from_argument('/path') is None - - def test_absolute_path_is_lv(self, volumes): - volumes.append(self.foo_volume) - assert api.get_lv_from_argument('/path/to/lv') == self.foo_volume - - class TestRemoveLV(object): def test_removes_lv(self, monkeypatch): @@ -548,18 +190,18 @@ class TestCreateLV(object): @patch('ceph_volume.api.lvm.process.run') @patch('ceph_volume.api.lvm.process.call') - @patch('ceph_volume.api.lvm.get_lv') - def test_uses_size(self, m_get_lv, m_call, m_run, monkeypatch): - m_get_lv.return_value = self.foo_volume + @patch('ceph_volume.api.lvm.get_first_lv') + def test_uses_size(self, m_get_first_lv, m_call, m_run, monkeypatch): + m_get_first_lv.return_value = self.foo_volume api.create_lv('foo', 0, vg=self.foo_group, size=5368709120, tags={'ceph.type': 'data'}) expected = ['lvcreate', '--yes', '-l', '1280', '-n', 'foo-0', 'foo_group'] m_run.assert_called_with(expected) @patch('ceph_volume.api.lvm.process.run') @patch('ceph_volume.api.lvm.process.call') - @patch('ceph_volume.api.lvm.get_lv') - def test_uses_extents(self, m_get_lv, m_call, m_run, monkeypatch): - m_get_lv.return_value = self.foo_volume + @patch('ceph_volume.api.lvm.get_first_lv') + def test_uses_extents(self, m_get_first_lv, m_call, m_run, monkeypatch): + m_get_first_lv.return_value = self.foo_volume api.create_lv('foo', 0, vg=self.foo_group, extents='50', tags={'ceph.type': 'data'}) expected = ['lvcreate', '--yes', '-l', '50', '-n', 'foo-0', 'foo_group'] m_run.assert_called_with(expected) @@ -569,18 +211,18 @@ class TestCreateLV(object): (3, 33),]) @patch('ceph_volume.api.lvm.process.run') @patch('ceph_volume.api.lvm.process.call') - @patch('ceph_volume.api.lvm.get_lv') - def test_uses_slots(self, m_get_lv, m_call, m_run, monkeypatch, test_input, expected): - m_get_lv.return_value = self.foo_volume + @patch('ceph_volume.api.lvm.get_first_lv') + def test_uses_slots(self, m_get_first_lv, m_call, m_run, monkeypatch, test_input, expected): + m_get_first_lv.return_value = self.foo_volume api.create_lv('foo', 0, vg=self.foo_group, slots=test_input, tags={'ceph.type': 'data'}) expected = ['lvcreate', '--yes', '-l', str(expected), '-n', 'foo-0', 'foo_group'] m_run.assert_called_with(expected) @patch('ceph_volume.api.lvm.process.run') @patch('ceph_volume.api.lvm.process.call') - @patch('ceph_volume.api.lvm.get_lv') - def test_uses_all(self, m_get_lv, m_call, m_run, monkeypatch): - m_get_lv.return_value = self.foo_volume + @patch('ceph_volume.api.lvm.get_first_lv') + def test_uses_all(self, m_get_first_lv, m_call, m_run, monkeypatch): + m_get_first_lv.return_value = self.foo_volume api.create_lv('foo', 0, vg=self.foo_group, tags={'ceph.type': 'data'}) expected = ['lvcreate', '--yes', '-l', '100%FREE', '-n', 'foo-0', 'foo_group'] m_run.assert_called_with(expected) @@ -588,9 +230,9 @@ class TestCreateLV(object): @patch('ceph_volume.api.lvm.process.run') @patch('ceph_volume.api.lvm.process.call') @patch('ceph_volume.api.lvm.Volume.set_tags') - @patch('ceph_volume.api.lvm.get_lv') - def test_calls_to_set_tags_default(self, m_get_lv, m_set_tags, m_call, m_run, monkeypatch): - m_get_lv.return_value = self.foo_volume + @patch('ceph_volume.api.lvm.get_first_lv') + def test_calls_to_set_tags_default(self, m_get_first_lv, m_set_tags, m_call, m_run, monkeypatch): + m_get_first_lv.return_value = self.foo_volume api.create_lv('foo', 0, vg=self.foo_group) tags = { "ceph.osd_id": "null", @@ -603,9 +245,9 @@ class TestCreateLV(object): @patch('ceph_volume.api.lvm.process.run') @patch('ceph_volume.api.lvm.process.call') @patch('ceph_volume.api.lvm.Volume.set_tags') - @patch('ceph_volume.api.lvm.get_lv') - def test_calls_to_set_tags_arg(self, m_get_lv, m_set_tags, m_call, m_run, monkeypatch): - m_get_lv.return_value = self.foo_volume + @patch('ceph_volume.api.lvm.get_first_lv') + def test_calls_to_set_tags_arg(self, m_get_first_lv, m_set_tags, m_call, m_run, monkeypatch): + m_get_first_lv.return_value = self.foo_volume api.create_lv('foo', 0, vg=self.foo_group, tags={'ceph.type': 'data'}) tags = { "ceph.type": "data", @@ -617,10 +259,10 @@ class TestCreateLV(object): @patch('ceph_volume.api.lvm.process.call') @patch('ceph_volume.api.lvm.get_device_vgs') @patch('ceph_volume.api.lvm.create_vg') - @patch('ceph_volume.api.lvm.get_lv') - def test_create_vg(self, m_get_lv, m_create_vg, m_get_device_vgs, m_call, + @patch('ceph_volume.api.lvm.get_first_lv') + def test_create_vg(self, m_get_first_lv, m_create_vg, m_get_device_vgs, m_call, m_run, monkeypatch): - m_get_lv.return_value = self.foo_volume + m_get_first_lv.return_value = self.foo_volume m_get_device_vgs.return_value = [] api.create_lv('foo', 0, device='dev/foo', size='5G', tags={'ceph.type': 'data'}) m_create_vg.assert_called_with('dev/foo', name_prefix='ceph') @@ -711,19 +353,19 @@ class TestExtendVG(object): self.foo_volume = api.VolumeGroup(vg_name='foo', lv_tags='') def test_uses_single_device_in_list(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.extend_vg(self.foo_volume, ['/dev/sda']) expected = ['vgextend', '--force', '--yes', 'foo', '/dev/sda'] assert fake_run.calls[0]['args'][0] == expected def test_uses_single_device(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.extend_vg(self.foo_volume, '/dev/sda') expected = ['vgextend', '--force', '--yes', 'foo', '/dev/sda'] assert fake_run.calls[0]['args'][0] == expected def test_uses_multiple_devices(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.extend_vg(self.foo_volume, ['/dev/sda', '/dev/sdb']) expected = ['vgextend', '--force', '--yes', 'foo', '/dev/sda', '/dev/sdb'] assert fake_run.calls[0]['args'][0] == expected @@ -735,19 +377,19 @@ class TestReduceVG(object): self.foo_volume = api.VolumeGroup(vg_name='foo', lv_tags='') def test_uses_single_device_in_list(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.reduce_vg(self.foo_volume, ['/dev/sda']) expected = ['vgreduce', '--force', '--yes', 'foo', '/dev/sda'] assert fake_run.calls[0]['args'][0] == expected def test_uses_single_device(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.reduce_vg(self.foo_volume, '/dev/sda') expected = ['vgreduce', '--force', '--yes', 'foo', '/dev/sda'] assert fake_run.calls[0]['args'][0] == expected def test_uses_multiple_devices(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.reduce_vg(self.foo_volume, ['/dev/sda', '/dev/sdb']) expected = ['vgreduce', '--force', '--yes', 'foo', '/dev/sda', '/dev/sdb'] assert fake_run.calls[0]['args'][0] == expected @@ -759,28 +401,28 @@ class TestCreateVG(object): self.foo_volume = api.VolumeGroup(vg_name='foo', lv_tags='') def test_no_name(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.create_vg('/dev/sda') result = fake_run.calls[0]['args'][0] assert '/dev/sda' in result assert result[-2].startswith('ceph-') def test_devices_list(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.create_vg(['/dev/sda', '/dev/sdb'], name='ceph') result = fake_run.calls[0]['args'][0] expected = ['vgcreate', '--force', '--yes', 'ceph', '/dev/sda', '/dev/sdb'] assert result == expected def test_name_prefix(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.create_vg('/dev/sda', name_prefix='master') result = fake_run.calls[0]['args'][0] assert '/dev/sda' in result assert result[-2].startswith('master-') def test_specific_name(self, monkeypatch, fake_run): - monkeypatch.setattr(api, 'get_vg', lambda **kw: True) + monkeypatch.setattr(api, 'get_first_vg', lambda **kw: True) api.create_vg('/dev/sda', name='master') result = fake_run.calls[0]['args'][0] assert '/dev/sda' in result @@ -947,30 +589,6 @@ class TestSplitNameParser(object): assert '/dev/mapper' not in result['VG_NAME'] -class TestIsLV(object): - - def test_is_not_an_lv(self, monkeypatch): - monkeypatch.setattr(api.process, 'call', lambda x, **kw: ('', '', 0)) - monkeypatch.setattr(api, 'dmsetup_splitname', lambda x, **kw: {}) - assert api.is_lv('/dev/sda1', lvs=[]) is False - - def test_lvs_not_found(self, monkeypatch, volumes): - CephVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', lv_tags="ceph.type=data") - volumes.append(CephVolume) - splitname = {'LV_NAME': 'data', 'VG_NAME': 'ceph'} - monkeypatch.setattr(api, 'dmsetup_splitname', lambda x, **kw: splitname) - assert api.is_lv('/dev/sda1', lvs=volumes) is False - - def test_is_lv(self, monkeypatch, volumes): - CephVolume = api.Volume( - vg_name='ceph', lv_name='data', - lv_path='/dev/vg/foo', lv_tags="ceph.type=data" - ) - volumes.append(CephVolume) - splitname = {'LV_NAME': 'data', 'VG_NAME': 'ceph'} - monkeypatch.setattr(api, 'dmsetup_splitname', lambda x, **kw: splitname) - assert api.is_lv('/dev/sda1', lvs=volumes) is True - class TestGetDeviceVgs(object): @patch('ceph_volume.process.call') @@ -990,3 +608,239 @@ class TestGetDeviceLvs(object): pcall.return_value = ('', '', '') vgs = api.get_device_lvs('/dev/foo') assert vgs == [] + + +# NOTE: api.convert_filters_to_str() and api.convert_tags_to_str() should get +# tested automatically while testing api.make_filters_lvmcmd_ready() +class TestMakeFiltersLVMCMDReady(object): + + def test_with_no_filters_and_no_tags(self): + retval = api.make_filters_lvmcmd_ready(None, None) + + assert isinstance(retval, str) + assert retval == '' + + def test_with_filters_and_no_tags(self): + filters = {'lv_name': 'lv1', 'lv_path': '/dev/sda'} + + retval = api.make_filters_lvmcmd_ready(filters, None) + + assert isinstance(retval, str) + for k, v in filters.items(): + assert k in retval + assert v in retval + + def test_with_no_filters_and_with_tags(self): + tags = {'ceph.type': 'data', 'ceph.osd_id': '0'} + + retval = api.make_filters_lvmcmd_ready(None, tags) + + assert isinstance(retval, str) + assert 'tags' in retval + for k, v in tags.items(): + assert k in retval + assert v in retval + assert retval.find('tags') < retval.find(k) < retval.find(v) + + def test_with_filters_and_tags(self): + filters = {'lv_name': 'lv1', 'lv_path': '/dev/sda'} + tags = {'ceph.type': 'data', 'ceph.osd_id': '0'} + + retval = api.make_filters_lvmcmd_ready(filters, tags) + + assert isinstance(retval, str) + for f, t in zip(filters.items(), tags.items()): + assert f[0] in retval + assert f[1] in retval + assert t[0] in retval + assert t[1] in retval + assert retval.find(f[0]) < retval.find(f[1]) < \ + retval.find('tags') < retval.find(t[0]) < retval.find(t[1]) + + +class TestGetPVs(object): + + def test_get_pvs(self, monkeypatch): + pv1 = api.PVolume(pv_name='/dev/sda', pv_uuid='0000', pv_tags={}, + vg_name='vg1') + pv2 = api.PVolume(pv_name='/dev/sdb', pv_uuid='0001', pv_tags={}, + vg_name='vg2') + pvs = [pv1, pv2] + stdout = ['{};{};{};{};;'.format(pv1.pv_name, pv1.pv_tags, pv1.pv_uuid, pv1.vg_name), + '{};{};{};{};;'.format(pv2.pv_name, pv2.pv_tags, pv2.pv_uuid, pv2.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + pvs_ = api.get_pvs() + assert len(pvs_) == len(pvs) + for pv, pv_ in zip(pvs, pvs_): + assert pv_.pv_name == pv.pv_name + + def test_get_pvs_single_pv(self, monkeypatch): + pv1 = api.PVolume(pv_name='/dev/sda', pv_uuid='0000', pv_tags={}, + vg_name='vg1') + pvs = [pv1] + stdout = ['{};;;;;;'.format(pv1.pv_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + pvs_ = api.get_pvs() + assert len(pvs_) == 1 + assert pvs_[0].pv_name == pvs[0].pv_name + + def test_get_pvs_empty(self, monkeypatch): + monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('', '', 0)) + assert api.get_pvs() == [] + + +class TestGetVGs(object): + + def test_get_vgs(self, monkeypatch): + vg1 = api.VolumeGroup(vg_name='vg1') + vg2 = api.VolumeGroup(vg_name='vg2') + vgs = [vg1, vg2] + stdout = ['{};;;;;;'.format(vg1.vg_name), + '{};;;;;;'.format(vg2.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + vgs_ = api.get_vgs() + assert len(vgs_) == len(vgs) + for vg, vg_ in zip(vgs, vgs_): + assert vg_.vg_name == vg.vg_name + + def test_get_vgs_single_vg(self, monkeypatch): + vg1 = api.VolumeGroup(vg_name='vg'); vgs = [vg1] + stdout = ['{};;;;;;'.format(vg1.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + vgs_ = api.get_vgs() + assert len(vgs_) == 1 + assert vgs_[0].vg_name == vgs[0].vg_name + + def test_get_vgs_empty(self, monkeypatch): + monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('', '', 0)) + assert api.get_vgs() == [] + + +class TestGetLVs(object): + + def test_get_lvs(self, monkeypatch): + lv1 = api.Volume(lv_tags='ceph.type=data', lv_path='/dev/vg1/lv1', + lv_name='lv1', vg_name='vg1') + lv2 = api.Volume(lv_tags='ceph.type=data', lv_path='/dev/vg2/lv2', + lv_name='lv2', vg_name='vg2') + lvs = [lv1, lv2] + stdout = ['{};{};{};{}'.format(lv1.lv_tags, lv1.lv_path, lv1.lv_name, + lv1.vg_name), + '{};{};{};{}'.format(lv2.lv_tags, lv2.lv_path, lv2.lv_name, + lv2.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + lvs_ = api.get_lvs() + assert len(lvs_) == len(lvs) + for lv, lv_ in zip(lvs, lvs_): + assert lv.__dict__ == lv_.__dict__ + + def test_get_lvs_single_lv(self, monkeypatch): + stdout = ['ceph.type=data;/dev/vg/lv;lv;vg'] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + lvs = [] + lvs.append((api.Volume(lv_tags='ceph.type=data', + lv_path='/dev/vg/lv', + lv_name='lv', vg_name='vg'))) + + lvs_ = api.get_lvs() + assert len(lvs_) == len(lvs) + assert lvs[0].__dict__ == lvs_[0].__dict__ + + def test_get_lvs_empty(self, monkeypatch): + monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('', '', 0)) + assert api.get_lvs() == [] + + +class TestGetFirstPV(object): + + def test_get_first_pv(self, monkeypatch): + pv1 = api.PVolume(pv_name='/dev/sda', pv_uuid='0000', pv_tags={}, + vg_name='vg1') + pv2 = api.PVolume(pv_name='/dev/sdb', pv_uuid='0001', pv_tags={}, + vg_name='vg2') + stdout = ['{};{};{};{};;'.format(pv1.pv_name, pv1.pv_tags, pv1.pv_uuid, pv1.vg_name), + '{};{};{};{};;'.format(pv2.pv_name, pv2.pv_tags, pv2.pv_uuid, pv2.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + pv_ = api.get_first_pv() + assert isinstance(pv_, api.PVolume) + assert pv_.pv_name == pv1.pv_name + + def test_get_first_pv_single_pv(self, monkeypatch): + pv = api.PVolume(pv_name='/dev/sda', pv_uuid='0000', pv_tags={}, + vg_name='vg1') + stdout = ['{};;;;;;'.format(pv.pv_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + pv_ = api.get_first_pv() + assert isinstance(pv_, api.PVolume) + assert pv_.pv_name == pv.pv_name + + def test_get_first_pv_empty(self, monkeypatch): + monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('', '', 0)) + assert api.get_first_pv() == [] + + +class TestGetFirstVG(object): + + def test_get_first_vg(self, monkeypatch): + vg1 = api.VolumeGroup(vg_name='vg1') + vg2 = api.VolumeGroup(vg_name='vg2') + stdout = ['{};;;;;;'.format(vg1.vg_name), '{};;;;;;'.format(vg2.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + vg_ = api.get_first_vg() + assert isinstance(vg_, api.VolumeGroup) + assert vg_.vg_name == vg1.vg_name + + def test_get_first_vg_single_vg(self, monkeypatch): + vg = api.VolumeGroup(vg_name='vg') + stdout = ['{};;;;;;'.format(vg.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + vg_ = api.get_first_vg() + assert isinstance(vg_, api.VolumeGroup) + assert vg_.vg_name == vg.vg_name + + def test_get_first_vg_empty(self, monkeypatch): + monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('', '', 0)) + vg_ = api.get_first_vg() + assert vg_ == [] + + +class TestGetFirstLV(object): + + def test_get_first_lv(self, monkeypatch): + lv1 = api.Volume(lv_tags='ceph.type=data', lv_path='/dev/vg1/lv1', + lv_name='lv1', vg_name='vg1') + lv2 = api.Volume(lv_tags='ceph.type=data', lv_path='/dev/vg2/lv2', + lv_name='lv2', vg_name='vg2') + stdout = ['{};{};{};{}'.format(lv1.lv_tags, lv1.lv_path, lv1.lv_name, + lv1.vg_name), + '{};{};{};{}'.format(lv2.lv_tags, lv2.lv_path, lv2.lv_name, + lv2.vg_name)] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + + lv_ = api.get_first_lv() + assert isinstance(lv_, api.Volume) + assert lv_.lv_name == lv1.lv_name + + def test_get_first_lv_single_lv(self, monkeypatch): + stdout = ['ceph.type=data;/dev/vg/lv;lv;vg'] + monkeypatch.setattr(api.process, 'call', lambda x,**kw: (stdout, '', 0)) + lv = api.Volume(lv_tags='ceph.type=data', + lv_path='/dev/vg/lv', + lv_name='lv', vg_name='vg') + + lv_ = api.get_first_lv() + assert isinstance(lv_, api.Volume) + assert lv_.lv_name == lv.lv_name + + def test_get_first_lv_empty(self, monkeypatch): + monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('', '', 0)) + assert api.get_lvs() == [] diff --git a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py index 74985e253..195082e7c 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py @@ -3,7 +3,6 @@ import pytest from mock.mock import patch, PropertyMock from ceph_volume.util import disk from ceph_volume.util.constants import ceph_disk_guids -from ceph_volume.api import lvm as lvm_api from ceph_volume import conf, configuration @@ -139,49 +138,6 @@ def conf_ceph_stub(monkeypatch, tmpfile): return apply -@pytest.fixture -def volumes(monkeypatch): - monkeypatch.setattr('ceph_volume.process.call', lambda x, **kw: ('', '', 0)) - volumes = lvm_api.Volumes() - volumes._purge() - return volumes - - -@pytest.fixture -def volume_groups(monkeypatch): - monkeypatch.setattr('ceph_volume.process.call', lambda x, **kw: ('', '', 0)) - vgs = lvm_api.VolumeGroups() - vgs._purge() - return vgs - -def volume_groups_empty(monkeypatch): - monkeypatch.setattr('ceph_volume.process.call', lambda x, **kw: ('', '', 0)) - vgs = lvm_api.VolumeGroups(populate=False) - return vgs - -@pytest.fixture -def stub_vgs(monkeypatch, volume_groups): - def apply(vgs): - monkeypatch.setattr(lvm_api, 'get_api_vgs', lambda: vgs) - return apply - - -# TODO: allow init-ing pvolumes to list we want -@pytest.fixture -def pvolumes(monkeypatch): - monkeypatch.setattr('ceph_volume.process.call', lambda x, **kw: ('', '', 0)) - pvolumes = lvm_api.PVolumes() - pvolumes._purge() - return pvolumes - -@pytest.fixture -def pvolumes_empty(monkeypatch): - monkeypatch.setattr('ceph_volume.process.call', lambda x, **kw: ('', '', 0)) - pvolumes = lvm_api.PVolumes(populate=False) - return pvolumes - - - @pytest.fixture def is_root(monkeypatch): """ @@ -215,15 +171,6 @@ def disable_kernel_queries(monkeypatch): monkeypatch.setattr("ceph_volume.util.disk.udevadm_property", lambda *a, **kw: {}) -@pytest.fixture -def disable_lvm_queries(monkeypatch): - ''' - This speeds up calls to Device and Disk - ''' - monkeypatch.setattr("ceph_volume.util.device.lvm.get_lv_from_argument", lambda path: None) - monkeypatch.setattr("ceph_volume.util.device.lvm.get_lv", lambda vg_name, lv_uuid: None) - - @pytest.fixture(params=[ '', 'ceph data', 'ceph journal', 'ceph block', 'ceph block.wal', 'ceph block.db', 'ceph lockbox']) @@ -290,12 +237,10 @@ def device_info(monkeypatch, patch_bluestore_label): monkeypatch.setattr("ceph_volume.sys_info.devices", {}) monkeypatch.setattr("ceph_volume.util.device.disk.get_devices", lambda: devices) if not devices: - monkeypatch.setattr("ceph_volume.util.device.lvm.get_lv_from_argument", lambda path: lv) + monkeypatch.setattr("ceph_volume.util.device.lvm.get_first_lv", lambda filters: lv) else: - monkeypatch.setattr("ceph_volume.util.device.lvm.get_lv_from_argument", lambda path: None) monkeypatch.setattr("ceph_volume.util.device.lvm.get_device_lvs", lambda path: [lv]) - monkeypatch.setattr("ceph_volume.util.device.lvm.get_lv", lambda vg_name, lv_uuid: lv) monkeypatch.setattr("ceph_volume.util.device.disk.lsblk", lambda path: lsblk) monkeypatch.setattr("ceph_volume.util.device.disk.blkid", lambda path: blkid) monkeypatch.setattr("ceph_volume.util.disk.udevadm_property", lambda *a, **kw: udevadm) diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_bluestore.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_bluestore.py index d64a7a568..ba54ea54d 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_bluestore.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_bluestore.py @@ -61,7 +61,6 @@ class TestMixedType(object): block_db_size=None, block_wal_size=None, osd_ids=[]) monkeypatch.setattr(lvm, 'VolumeGroup', lambda x, **kw: []) - monkeypatch.setattr(lvm, 'VolumeGroups', lambda: []) bluestore.MixedType(args, [], [db_dev], []) @@ -69,7 +68,7 @@ class TestMixedTypeConfiguredSize(object): # uses a block.db size that has been configured via ceph.conf, instead of # defaulting to 'as large as possible' - def test_hdd_device_is_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_hdd_device_is_large_enough(self, fakedevice, factory, conf_ceph): # 3GB block.db in ceph.conf conf_ceph(get_safe=lambda *a: 3147483640) args = factory(filtered_devices=[], osds_per_device=1, @@ -87,7 +86,7 @@ class TestMixedTypeConfiguredSize(object): assert osd['block.db']['path'] == 'vg: vg/lv' assert osd['block.db']['percentage'] == 100 - def test_ssd_device_is_not_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_ssd_device_is_not_large_enough(self, fakedevice, factory, conf_ceph): # 7GB block.db in ceph.conf conf_ceph(get_safe=lambda *a: 7747483640) args = factory(filtered_devices=[], osds_per_device=1, @@ -102,7 +101,7 @@ class TestMixedTypeConfiguredSize(object): expected = 'Not enough space in fast devices (5.66 GB) to create 1 x 7.22 GB block.db LV' assert expected in str(error.value) - def test_multi_hdd_device_is_not_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_multi_hdd_device_is_not_large_enough(self, fakedevice, factory, conf_ceph): # 3GB block.db in ceph.conf conf_ceph(get_safe=lambda *a: 3147483640) args = factory(filtered_devices=[], osds_per_device=2, @@ -120,7 +119,7 @@ class TestMixedTypeConfiguredSize(object): class TestMixedTypeLargeAsPossible(object): - def test_hdd_device_is_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_hdd_device_is_large_enough(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: None) args = factory(filtered_devices=[], osds_per_device=1, block_db_size=None, block_wal_size=None, @@ -138,7 +137,7 @@ class TestMixedTypeLargeAsPossible(object): # as large as possible assert osd['block.db']['percentage'] == 100 - def test_multi_hdd_device_is_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_multi_hdd_device_is_large_enough(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: None) args = factory(filtered_devices=[], osds_per_device=2, block_db_size=None, block_wal_size=None, @@ -156,7 +155,7 @@ class TestMixedTypeLargeAsPossible(object): # as large as possible assert osd['block.db']['percentage'] == 50 - def test_multi_hdd_device_is_not_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_multi_hdd_device_is_not_large_enough(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: None) args = factory(filtered_devices=[], osds_per_device=2, block_db_size=None, block_wal_size=None, @@ -173,7 +172,7 @@ class TestMixedTypeLargeAsPossible(object): class TestMixedTypeWithExplicitDevices(object): - def test_multi_hdd_device_is_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_multi_hdd_device_is_large_enough(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: None) args = factory(filtered_devices=[], osds_per_device=2, block_db_size=None, block_wal_size=None, @@ -190,7 +189,7 @@ class TestMixedTypeWithExplicitDevices(object): # as large as possible assert osd['block.wal']['percentage'] == 50 - def test_wal_device_is_not_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_wal_device_is_not_large_enough(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: None) args = factory(filtered_devices=[], osds_per_device=2, block_db_size=None, block_wal_size=None, diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_filestore.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_filestore.py index 5bfc07086..4fd94f5b7 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_filestore.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/strategies/test_filestore.py @@ -118,7 +118,7 @@ class TestSingleType(object): class TestMixedType(object): - def test_minimum_size_is_not_met(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_minimum_size_is_not_met(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: '120') args = factory(filtered_devices=[], osds_per_device=1, journal_size=None, osd_ids=[]) @@ -131,7 +131,7 @@ class TestMixedType(object): msg = "journal sizes must be larger than 2GB, detected: 120.00 MB" assert msg in str(error.value) - def test_ssd_device_is_not_large_enough(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_ssd_device_is_not_large_enough(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: '7120') args = factory(filtered_devices=[], osds_per_device=1, journal_size=None, osd_ids=[]) @@ -144,7 +144,7 @@ class TestMixedType(object): msg = "Not enough space in fast devices (5.66 GB) to create 1 x 6.95 GB journal LV" assert msg in str(error.value) - def test_hdd_device_is_lvm_member_fails(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_hdd_device_is_lvm_member_fails(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: '5120') args = factory(filtered_devices=[], osds_per_device=1, journal_size=None, osd_ids=[]) @@ -159,7 +159,6 @@ class TestMixedType(object): @patch('ceph_volume.devices.lvm.strategies.strategies.MixedStrategy.get_common_vg') def test_ssd_is_lvm_member_doesnt_fail(self, patched_get_common_vg, - volumes, fakedevice, factory, conf_ceph): @@ -178,13 +177,15 @@ class TestMixedType(object): args = factory(filtered_devices=[], osds_per_device=1, journal_size=None, osd_ids=[]) devices = [ssd, hdd] - result = filestore.MixedType.with_auto_devices(args, devices).computed['osds'][0] + + result = filestore.MixedType.with_auto_devices(args, devices).\ + computed['osds'][0] assert result['journal']['path'] == 'vg: fast' assert result['journal']['percentage'] == 71 assert result['journal']['human_readable_size'] == '5.00 GB' @patch('ceph_volume.api.lvm.get_device_vgs') - def test_no_common_vg(self, patched_get_device_vgs, volumes, fakedevice, factory, conf_ceph): + def test_no_common_vg(self, patched_get_device_vgs, fakedevice, factory, conf_ceph): patched_get_device_vgs.side_effect = lambda x: [lvm.VolumeGroup(vg_name='{}'.format(x[-1]), vg_tags='')] ssd1 = fakedevice( used_by_ceph=False, is_lvm_member=True, rotational=False, sys_api=dict(size=6073740000) @@ -195,14 +196,15 @@ class TestMixedType(object): hdd = fakedevice(used_by_ceph=False, is_lvm_member=False, rotational=True, sys_api=dict(size=6073740000)) conf_ceph(get_safe=lambda *a: '5120') - args = factory(filtered_devices=[], osds_per_device=1, - journal_size=None, osd_ids=[]) + args = factory(filtered_devices=[], osds_per_device=1, osd_ids=[], + journal_size=None) devices = [ssd1, ssd2, hdd] + with pytest.raises(RuntimeError) as error: filestore.MixedType.with_auto_devices(args, devices) assert 'Could not find a common VG between devices' in str(error.value) - def test_ssd_device_fails_multiple_osds(self, stub_vgs, fakedevice, factory, conf_ceph): + def test_ssd_device_fails_multiple_osds(self, fakedevice, factory, conf_ceph): conf_ceph(get_safe=lambda *a: '15120') args = factory(filtered_devices=[], osds_per_device=2, journal_size=None, osd_ids=[]) diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py index cfa7de8e8..33e0ed32b 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py @@ -1,4 +1,5 @@ import pytest +from copy import deepcopy from ceph_volume.devices.lvm import activate from ceph_volume.api import lvm as api from ceph_volume.tests.conftest import Capture @@ -22,34 +23,42 @@ class TestActivate(object): # test the negative side effect with an actual functional run, so we must # setup a perfect scenario for this test to check it can really work # with/without osd_id - def test_no_osd_id_matches_fsid(self, is_root, volumes, monkeypatch, capture): - FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', lv_tags="ceph.osd_fsid=1234") + def test_no_osd_id_matches_fsid(self, is_root, monkeypatch, capture): + FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', + lv_tags="ceph.osd_fsid=1234") + volumes = [] volumes.append(FooVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: volumes) monkeypatch.setattr(activate, 'activate_filestore', capture) args = Args(osd_id=None, osd_fsid='1234', filestore=True) activate.Activate([]).activate(args) assert capture.calls[0]['args'][0] == [FooVolume] - def test_no_osd_id_matches_fsid_bluestore(self, is_root, volumes, monkeypatch, capture): - FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', lv_tags="ceph.osd_fsid=1234") + def test_no_osd_id_matches_fsid_bluestore(self, is_root, monkeypatch, capture): + FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', + lv_tags="ceph.osd_fsid=1234") + volumes = [] volumes.append(FooVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: volumes) monkeypatch.setattr(activate, 'activate_bluestore', capture) args = Args(osd_id=None, osd_fsid='1234', bluestore=True) activate.Activate([]).activate(args) assert capture.calls[0]['args'][0] == [FooVolume] - def test_no_osd_id_no_matching_fsid(self, is_root, volumes, monkeypatch, capture): - FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', lv_tags="ceph.osd_fsid=11234") + def test_no_osd_id_no_matching_fsid(self, is_root, monkeypatch, capture): + FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', + lv_tags="ceph.osd_fsid=1111") + volumes = [] volumes.append(FooVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: []) + monkeypatch.setattr(api, 'get_first_lv', lambda **kwargs: []) monkeypatch.setattr(activate, 'activate_filestore', capture) - args = Args(osd_id=None, osd_fsid='1234') + + args = Args(osd_id=None, osd_fsid='2222') with pytest.raises(RuntimeError): activate.Activate([]).activate(args) - def test_filestore_no_systemd(self, is_root, volumes, monkeypatch, capture): + def test_filestore_no_systemd(self, is_root, monkeypatch, capture): monkeypatch.setattr('ceph_volume.configuration.load', lambda: None) fake_enable = Capture() fake_start_osd = Capture() @@ -70,16 +79,21 @@ class TestActivate(object): DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/journal,ceph.journal_uuid=000,ceph.type=data,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_uuid='001', + lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/" + \ + "journal,ceph.journal_uuid=000,ceph.type=data," + \ + "ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) volumes.append(JournalVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, filestore=True) activate.Activate([]).activate(args) assert fake_enable.calls == [] assert fake_start_osd.calls == [] - def test_filestore_no_systemd_autodetect(self, is_root, volumes, monkeypatch, capture): + def test_filestore_no_systemd_autodetect(self, is_root, monkeypatch, capture): monkeypatch.setattr('ceph_volume.configuration.load', lambda: None) fake_enable = Capture() fake_start_osd = Capture() @@ -100,16 +114,22 @@ class TestActivate(object): DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/journal,ceph.journal_uuid=000,ceph.type=data,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_uuid='001', + lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/" + \ + "journal,ceph.journal_uuid=000,ceph.type=data," + \ + "ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) volumes.append(JournalVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, filestore=True, auto_detect_objectstore=True) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + + args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, + filestore=True, auto_detect_objectstore=True) activate.Activate([]).activate(args) assert fake_enable.calls == [] assert fake_start_osd.calls == [] - def test_filestore_systemd_autodetect(self, is_root, volumes, monkeypatch, capture): + def test_filestore_systemd_autodetect(self, is_root, monkeypatch, capture): fake_enable = Capture() fake_start_osd = Capture() monkeypatch.setattr('ceph_volume.configuration.load', lambda: None) @@ -130,16 +150,22 @@ class TestActivate(object): DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/journal,ceph.journal_uuid=000,ceph.type=data,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_uuid='001', + lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/" + \ + "journal,ceph.journal_uuid=000,ceph.type=data," + \ + "ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) volumes.append(JournalVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, filestore=True, auto_detect_objectstore=False) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + + args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, + filestore=True, auto_detect_objectstore=False) activate.Activate([]).activate(args) assert fake_enable.calls != [] assert fake_start_osd.calls != [] - def test_filestore_systemd(self, is_root, volumes, monkeypatch, capture): + def test_filestore_systemd(self, is_root, monkeypatch, capture): fake_enable = Capture() fake_start_osd = Capture() monkeypatch.setattr('ceph_volume.configuration.load', lambda: None) @@ -160,16 +186,22 @@ class TestActivate(object): DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/journal,ceph.journal_uuid=000,ceph.type=data,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_uuid='001', + lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/" + \ + "journal,ceph.journal_uuid=000,ceph.type=data," + \ + "ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) volumes.append(JournalVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, filestore=True) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + + args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, + filestore=True) activate.Activate([]).activate(args) assert fake_enable.calls != [] assert fake_start_osd.calls != [] - def test_bluestore_no_systemd(self, is_root, volumes, monkeypatch, capture): + def test_bluestore_no_systemd(self, is_root, monkeypatch, capture): fake_enable = Capture() fake_start_osd = Capture() monkeypatch.setattr('ceph_volume.util.system.path_is_mounted', lambda *a, **kw: True) @@ -180,15 +212,18 @@ class TestActivate(object): DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000,ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000," + \ + "ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, bluestore=True) activate.Activate([]).activate(args) assert fake_enable.calls == [] assert fake_start_osd.calls == [] - def test_bluestore_systemd(self, is_root, volumes, monkeypatch, capture): + def test_bluestore_systemd(self, is_root, monkeypatch, capture): fake_enable = Capture() fake_start_osd = Capture() monkeypatch.setattr('ceph_volume.util.system.path_is_mounted', lambda *a, **kw: True) @@ -199,15 +234,19 @@ class TestActivate(object): DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000,ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000," + \ + "ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, bluestore=True) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + + args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, + bluestore=True) activate.Activate([]).activate(args) assert fake_enable.calls != [] assert fake_start_osd.calls != [] - def test_bluestore_no_systemd_autodetect(self, is_root, volumes, monkeypatch, capture): + def test_bluestore_no_systemd_autodetect(self, is_root, monkeypatch, capture): fake_enable = Capture() fake_start_osd = Capture() monkeypatch.setattr('ceph_volume.util.system.path_is_mounted', lambda *a, **kw: True) @@ -218,29 +257,39 @@ class TestActivate(object): DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,,ceph.block_uuid=000,ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_tags="ceph.cluster_name=ceph,,ceph.block_uuid=000," + \ + "ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, bluestore=True, auto_detect_objectstore=True) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + + args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, + bluestore=True, auto_detect_objectstore=True) activate.Activate([]).activate(args) assert fake_enable.calls == [] assert fake_start_osd.calls == [] - def test_bluestore_systemd_autodetect(self, is_root, volumes, monkeypatch, capture): + def test_bluestore_systemd_autodetect(self, is_root, monkeypatch, capture): fake_enable = Capture() fake_start_osd = Capture() - monkeypatch.setattr('ceph_volume.util.system.path_is_mounted', lambda *a, **kw: True) - monkeypatch.setattr('ceph_volume.util.system.chown', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.util.system.path_is_mounted', + lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.util.system.chown', lambda *a, **kw: + True) monkeypatch.setattr('ceph_volume.process.run', lambda *a, **kw: True) monkeypatch.setattr(activate.systemctl, 'enable_volume', fake_enable) monkeypatch.setattr(activate.systemctl, 'start_osd', fake_start_osd) DataVolume = api.Volume( lv_name='data', lv_path='/dev/vg/data', - lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000,ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000," + \ + "ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes = [] volumes.append(DataVolume) - monkeypatch.setattr(api, 'Volumes', lambda: volumes) - args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, bluestore=True, auto_detect_objectstore=False) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: deepcopy(volumes)) + + args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, + bluestore=True, auto_detect_objectstore=False) activate.Activate([]).activate(args) assert fake_enable.calls != [] assert fake_start_osd.calls != [] diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py index 1c3c25f67..06a37c490 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py @@ -1,4 +1,5 @@ import pytest +import json from ceph_volume.devices.lvm import batch @@ -133,7 +134,7 @@ class TestFilterDevices(object): abspath="/dev/sda", rotational=True, is_lvm_member=False, - available=True + available=True, ) ssd1 = factory( used_by_ceph=True, @@ -143,9 +144,34 @@ class TestFilterDevices(object): available=False ) args = factory(devices=[hdd1], db_devices=[ssd1], filtered_devices={}, - yes=True) + yes=True, format="", report=False) b = batch.Batch([]) b.args = args with pytest.raises(RuntimeError) as ex: b._filter_devices() assert '1 devices were filtered in non-interactive mode, bailing out' in str(ex.value) + + def test_no_auto_prints_json_on_unavailable_device_and_report(self, factory, capsys): + hdd1 = factory( + used_by_ceph=False, + abspath="/dev/sda", + rotational=True, + is_lvm_member=False, + available=True, + ) + ssd1 = factory( + used_by_ceph=True, + abspath="/dev/nvme0n1", + rotational=False, + is_lvm_member=True, + available=False + ) + captured = capsys.readouterr() + args = factory(devices=[hdd1], db_devices=[ssd1], filtered_devices={}, + yes=True, format="json", report=True) + b = batch.Batch([]) + b.args = args + with pytest.raises(SystemExit): + b._filter_devices() + result = json.loads(captured.out) + assert not result["changed"] diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py index e41cbba72..cf4b68c71 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py @@ -62,36 +62,35 @@ class TestPrettyReport(object): class TestList(object): - def test_empty_full_json_zero_exit_status(self, is_root, volumes, - factory, capsys): + def test_empty_full_json_zero_exit_status(self, is_root,factory,capsys): args = factory(format='json', device=None) lvm.listing.List([]).list(args) stdout, stderr = capsys.readouterr() assert stdout == '{}\n' - def test_empty_device_json_zero_exit_status(self, is_root, volumes, - factory, capsys): + def test_empty_device_json_zero_exit_status(self, is_root,factory,capsys): args = factory(format='json', device='/dev/sda1') lvm.listing.List([]).list(args) stdout, stderr = capsys.readouterr() assert stdout == '{}\n' - def test_empty_full_zero_exit_status(self, is_root, volumes, factory): + def test_empty_full_zero_exit_status(self, is_root, factory): args = factory(format='pretty', device=None) with pytest.raises(SystemExit): lvm.listing.List([]).list(args) - def test_empty_device_zero_exit_status(self, is_root, volumes, factory): + def test_empty_device_zero_exit_status(self, is_root, factory): args = factory(format='pretty', device='/dev/sda1') with pytest.raises(SystemExit): lvm.listing.List([]).list(args) class TestFullReport(object): - def test_no_ceph_lvs(self, volumes, monkeypatch): + def test_no_ceph_lvs(self, monkeypatch): # ceph lvs are detected by looking into its tags osd = api.Volume(lv_name='volume1', lv_path='/dev/VolGroup/lv', lv_tags={}) + volumes = [] volumes.append(osd) monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs: volumes) @@ -99,23 +98,22 @@ class TestFullReport(object): result = lvm.listing.List([]).full_report() assert result == {} - def test_ceph_data_lv_reported(self, pvolumes, volumes, monkeypatch): + def test_ceph_data_lv_reported(self, monkeypatch): tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' pv = api.PVolume(pv_name='/dev/sda1', pv_tags={}, pv_uuid="0000", vg_name='VolGroup', lv_uuid="aaaa") osd = api.Volume(lv_name='volume1', lv_uuid='y', lv_tags=tags, lv_path='/dev/VolGroup/lv', vg_name='VolGroup') - pvolumes.append(pv) + volumes = [] volumes.append(osd) - monkeypatch.setattr(lvm.listing.api, 'get_pvs', lambda **kwargs: - pvolumes) + monkeypatch.setattr(lvm.listing.api, 'get_first_pv', lambda **kwargs: pv) monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs: volumes) result = lvm.listing.List([]).full_report() assert result['0'][0]['name'] == 'volume1' - def test_ceph_journal_lv_reported(self, pvolumes, volumes, monkeypatch): + def test_ceph_journal_lv_reported(self, monkeypatch): tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' journal_tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=journal' pv = api.PVolume(pv_name='/dev/sda1', pv_tags={}, pv_uuid="0000", @@ -125,11 +123,10 @@ class TestFullReport(object): journal = api.Volume( lv_name='journal', lv_uuid='x', lv_tags=journal_tags, lv_path='/dev/VolGroup/journal', vg_name='VolGroup') - pvolumes.append(pv) + volumes = [] volumes.append(osd) volumes.append(journal) - monkeypatch.setattr(lvm.listing.api, 'get_pvs', lambda **kwargs: - pvolumes) + monkeypatch.setattr(lvm.listing.api,'get_first_pv',lambda **kwargs:pv) monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs: volumes) @@ -137,13 +134,14 @@ class TestFullReport(object): assert result['0'][0]['name'] == 'volume1' assert result['0'][1]['name'] == 'journal' - def test_ceph_wal_lv_reported(self, volumes, monkeypatch): + def test_ceph_wal_lv_reported(self, monkeypatch): tags = 'ceph.osd_id=0,ceph.wal_uuid=x,ceph.type=data' wal_tags = 'ceph.osd_id=0,ceph.wal_uuid=x,ceph.type=wal' osd = api.Volume(lv_name='volume1', lv_uuid='y', lv_tags=tags, lv_path='/dev/VolGroup/lv', vg_name='VolGroup') wal = api.Volume(lv_name='wal', lv_uuid='x', lv_tags=wal_tags, lv_path='/dev/VolGroup/wal', vg_name='VolGroup') + volumes = [] volumes.append(osd) volumes.append(wal) monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs: @@ -170,7 +168,7 @@ class TestFullReport(object): class TestSingleReport(object): - def test_not_a_ceph_lv(self, volumes, monkeypatch): + def test_not_a_ceph_lv(self, monkeypatch): # ceph lvs are detected by looking into its tags lv = api.Volume(lv_name='lv', lv_tags={}, lv_path='/dev/VolGroup/lv', vg_name='VolGroup') @@ -180,14 +178,13 @@ class TestSingleReport(object): result = lvm.listing.List([]).single_report('VolGroup/lv') assert result == {} - def test_report_a_ceph_lv(self, pvolumes, volumes, monkeypatch): + def test_report_a_ceph_lv(self, monkeypatch): # ceph lvs are detected by looking into its tags tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' lv = api.Volume(lv_name='lv', vg_name='VolGroup', lv_uuid='aaaa', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes = [] volumes.append(lv) - monkeypatch.setattr(lvm.listing.api, 'get_pvs', lambda **kwargs: - pvolumes) monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs: volumes) @@ -211,17 +208,23 @@ class TestSingleReport(object): assert result['0'][0]['type'] == 'journal' assert result['0'][0]['path'] == '/dev/sda1' - def test_report_a_ceph_lv_with_devices(self, volumes, pvolumes, monkeypatch): + def test_report_a_ceph_lv_with_devices(self, monkeypatch): + pvolumes = [] + tags = 'ceph.osd_id=0,ceph.type=data' pv1 = api.PVolume(vg_name="VolGroup", pv_name='/dev/sda1', pv_uuid='', pv_tags={}, lv_uuid="aaaa") pv2 = api.PVolume(vg_name="VolGroup", pv_name='/dev/sdb1', pv_uuid='', pv_tags={}, lv_uuid="aaaa") - lv = api.Volume(lv_name='lv', vg_name='VolGroup',lv_uuid='aaaa', - lv_path='/dev/VolGroup/lv', lv_tags=tags) pvolumes.append(pv1) pvolumes.append(pv2) + + + volumes = [] + lv = api.Volume(lv_name='lv', vg_name='VolGroup',lv_uuid='aaaa', + lv_path='/dev/VolGroup/lv', lv_tags=tags) volumes.append(lv) + monkeypatch.setattr(lvm.listing.api, 'get_pvs', lambda **kwargs: pvolumes) monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs: @@ -239,11 +242,11 @@ class TestSingleReport(object): assert result['0'][0]['path'] == '/dev/VolGroup/lv' assert result['0'][0]['devices'] == ['/dev/sda1', '/dev/sdb1'] - def test_report_a_ceph_lv_with_no_matching_devices(self, volumes, - monkeypatch): + def test_report_a_ceph_lv_with_no_matching_devices(self, monkeypatch): tags = 'ceph.osd_id=0,ceph.type=data' lv = api.Volume(lv_name='lv', vg_name='VolGroup', lv_uuid='aaaa', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes = [] volumes.append(lv) monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs: volumes) diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py index f16b2ffff..dd9028668 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py @@ -115,21 +115,6 @@ class TestPrepare(object): expected = 'skipping {}, it is already prepared'.format('/dev/sdfoo') assert expected in str(error.value) -class TestGetJournalLV(object): - - @pytest.mark.parametrize('arg', ['', '///', None, '/dev/sda1']) - def test_no_journal_on_invalid_path(self, monkeypatch, arg): - monkeypatch.setattr(lvm.prepare.api, 'get_lv', lambda **kw: False) - prepare = lvm.prepare.Prepare([]) - assert prepare.get_lv(arg) is None - - def test_no_journal_lv_found(self, monkeypatch): - # patch it with 0 so we know we are getting to get_lv - monkeypatch.setattr(lvm.prepare.api, 'get_lv', lambda **kw: 0) - prepare = lvm.prepare.Prepare([]) - assert prepare.get_lv('vg/lv') == 0 - - class TestActivate(object): def test_main_spits_help_with_no_arguments(self, capsys): diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_zap.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_zap.py index 20ca56b54..1fa22e5b6 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_zap.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_zap.py @@ -1,92 +1,119 @@ import os import pytest +from copy import deepcopy +from mock.mock import patch, call +from ceph_volume import process from ceph_volume.api import lvm as api from ceph_volume.devices.lvm import zap class TestFindAssociatedDevices(object): - def test_no_lvs_found_that_match_id(self, volumes, monkeypatch, device_info): - monkeypatch.setattr(zap.api, 'Volumes', lambda: volumes) + def test_no_lvs_found_that_match_id(self, monkeypatch, device_info): tags = 'ceph.osd_id=9,ceph.journal_uuid=x,ceph.type=data' - osd = api.Volume( - lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', vg_name='vg', lv_tags=tags) + osd = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_tags=tags, lv_path='/dev/VolGroup/lv') + volumes = [] volumes.append(osd) + monkeypatch.setattr(zap.api, 'get_lvs', lambda **kwargs: {}) + with pytest.raises(RuntimeError): zap.find_associated_devices(osd_id=10) - def test_no_lvs_found_that_match_fsid(self, volumes, monkeypatch, device_info): - monkeypatch.setattr(zap.api, 'Volumes', lambda: volumes) - tags = 'ceph.osd_id=9,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=data' - osd = api.Volume( - lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', vg_name='vg', lv_tags=tags) + def test_no_lvs_found_that_match_fsid(self, monkeypatch, device_info): + tags = 'ceph.osd_id=9,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,'+\ + 'ceph.type=data' + osd = api.Volume(lv_name='volume1', lv_uuid='y', lv_tags=tags, + vg_name='vg', lv_path='/dev/VolGroup/lv') + volumes = [] volumes.append(osd) + monkeypatch.setattr(zap.api, 'get_lvs', lambda **kwargs: {}) + with pytest.raises(RuntimeError): zap.find_associated_devices(osd_fsid='aaaa-lkjh') - def test_no_lvs_found_that_match_id_fsid(self, volumes, monkeypatch, device_info): - monkeypatch.setattr(zap.api, 'Volumes', lambda: volumes) - tags = 'ceph.osd_id=9,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=data' - osd = api.Volume( - lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', vg_name='vg', lv_tags=tags) + def test_no_lvs_found_that_match_id_fsid(self, monkeypatch, device_info): + tags = 'ceph.osd_id=9,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,'+\ + 'ceph.type=data' + osd = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_tags=tags, lv_path='/dev/VolGroup/lv') + volumes = [] volumes.append(osd) + monkeypatch.setattr(zap.api, 'get_lvs', lambda **kwargs: {}) + with pytest.raises(RuntimeError): zap.find_associated_devices(osd_id='9', osd_fsid='aaaa-lkjh') - def test_no_ceph_lvs_found(self, volumes, monkeypatch): - monkeypatch.setattr(zap.api, 'Volumes', lambda: volumes) - osd = api.Volume( - lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', lv_tags='') + def test_no_ceph_lvs_found(self, monkeypatch): + osd = api.Volume(lv_name='volume1', lv_uuid='y', lv_tags='', + lv_path='/dev/VolGroup/lv') + volumes = [] volumes.append(osd) + monkeypatch.setattr(zap.api, 'get_lvs', lambda **kwargs: {}) + with pytest.raises(RuntimeError): zap.find_associated_devices(osd_id=100) - def test_lv_is_matched_id(self, volumes, monkeypatch): - monkeypatch.setattr(zap.api, 'Volumes', lambda: volumes) + def test_lv_is_matched_id(self, monkeypatch): tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' - osd = api.Volume( - lv_name='volume1', lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/lv', lv_tags=tags) + osd = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='', + lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes = [] volumes.append(osd) + monkeypatch.setattr(zap.api, 'get_lvs', lambda **kw: volumes) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + result = zap.find_associated_devices(osd_id='0') assert result[0].abspath == '/dev/VolGroup/lv' - def test_lv_is_matched_fsid(self, volumes, monkeypatch): - monkeypatch.setattr(zap.api, 'Volumes', lambda: volumes) - tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=data' - osd = api.Volume( - lv_name='volume1', lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/lv', lv_tags=tags) + def test_lv_is_matched_fsid(self, monkeypatch): + tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,' +\ + 'ceph.type=data' + osd = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='', + lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes = [] volumes.append(osd) + monkeypatch.setattr(zap.api, 'get_lvs', lambda **kw: deepcopy(volumes)) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + result = zap.find_associated_devices(osd_fsid='asdf-lkjh') assert result[0].abspath == '/dev/VolGroup/lv' - def test_lv_is_matched_id_fsid(self, volumes, monkeypatch): - monkeypatch.setattr(zap.api, 'Volumes', lambda: volumes) - tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=data' - osd = api.Volume( - lv_name='volume1', lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/lv', lv_tags=tags) + def test_lv_is_matched_id_fsid(self, monkeypatch): + tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,' +\ + 'ceph.type=data' + osd = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='', + lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes = [] volumes.append(osd) + monkeypatch.setattr(zap.api, 'get_lvs', lambda **kw: volumes) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + result = zap.find_associated_devices(osd_id='0', osd_fsid='asdf-lkjh') assert result[0].abspath == '/dev/VolGroup/lv' class TestEnsureAssociatedLVs(object): - def test_nothing_is_found(self, volumes): + def test_nothing_is_found(self): + volumes = [] result = zap.ensure_associated_lvs(volumes) assert result == [] - def test_data_is_found(self, volumes): + def test_data_is_found(self): tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=data' osd = api.Volume( lv_name='volume1', lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/data', lv_tags=tags) + volumes = [] volumes.append(osd) result = zap.ensure_associated_lvs(volumes) assert result == ['/dev/VolGroup/data'] - def test_block_is_found(self, volumes): + def test_block_is_found(self): tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=block' osd = api.Volume( lv_name='volume1', lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/block', lv_tags=tags) + volumes = [] volumes.append(osd) result = zap.ensure_associated_lvs(volumes) assert result == ['/dev/VolGroup/block'] @@ -107,26 +134,29 @@ class TestEnsureAssociatedLVs(object): out, err = capsys.readouterr() assert "Zapping successful for OSD: 1" in err - def test_block_and_partition_are_found(self, volumes, monkeypatch): + def test_block_and_partition_are_found(self, monkeypatch): monkeypatch.setattr(zap.disk, 'get_device_from_partuuid', lambda x: '/dev/sdb1') tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=block' osd = api.Volume( lv_name='volume1', lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/block', lv_tags=tags) + volumes = [] volumes.append(osd) result = zap.ensure_associated_lvs(volumes) assert '/dev/sdb1' in result assert '/dev/VolGroup/block' in result - def test_journal_is_found(self, volumes): + def test_journal_is_found(self): tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=journal' osd = api.Volume( lv_name='volume1', lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes = [] volumes.append(osd) result = zap.ensure_associated_lvs(volumes) assert result == ['/dev/VolGroup/lv'] - def test_multiple_journals_are_found(self, volumes): + def test_multiple_journals_are_found(self): tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=journal' + volumes = [] for i in range(3): osd = api.Volume( lv_name='volume%s' % i, lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/lv%s' % i, lv_tags=tags) @@ -136,8 +166,9 @@ class TestEnsureAssociatedLVs(object): assert '/dev/VolGroup/lv1' in result assert '/dev/VolGroup/lv2' in result - def test_multiple_dbs_are_found(self, volumes): + def test_multiple_dbs_are_found(self): tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.journal_uuid=x,ceph.type=db' + volumes = [] for i in range(3): osd = api.Volume( lv_name='volume%s' % i, lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/lv%s' % i, lv_tags=tags) @@ -147,8 +178,9 @@ class TestEnsureAssociatedLVs(object): assert '/dev/VolGroup/lv1' in result assert '/dev/VolGroup/lv2' in result - def test_multiple_wals_are_found(self, volumes): + def test_multiple_wals_are_found(self): tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.wal_uuid=x,ceph.type=wal' + volumes = [] for i in range(3): osd = api.Volume( lv_name='volume%s' % i, lv_uuid='y', vg_name='', lv_path='/dev/VolGroup/lv%s' % i, lv_tags=tags) @@ -158,7 +190,8 @@ class TestEnsureAssociatedLVs(object): assert '/dev/VolGroup/lv1' in result assert '/dev/VolGroup/lv2' in result - def test_multiple_backing_devs_are_found(self, volumes): + def test_multiple_backing_devs_are_found(self): + volumes = [] for _type in ['journal', 'db', 'wal']: tags = 'ceph.osd_id=0,ceph.osd_fsid=asdf-lkjh,ceph.wal_uuid=x,ceph.type=%s' % _type osd = api.Volume( @@ -169,6 +202,16 @@ class TestEnsureAssociatedLVs(object): assert '/dev/VolGroup/lvwal' in result assert '/dev/VolGroup/lvdb' in result + @patch('ceph_volume.devices.lvm.zap.api.get_lvs') + def test_ensure_associated_lvs(self, m_get_lvs): + zap.ensure_associated_lvs([], lv_tags={'ceph.osd_id': '1'}) + calls = [ + call(tags={'ceph.type': 'journal', 'ceph.osd_id': '1'}), + call(tags={'ceph.type': 'db', 'ceph.osd_id': '1'}), + call(tags={'ceph.type': 'wal', 'ceph.osd_id': '1'}) + ] + m_get_lvs.assert_has_calls(calls, any_order=True) + class TestWipeFs(object): diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/__init__.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_prepare.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_prepare.py new file mode 100644 index 000000000..e4cf8ce11 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_prepare.py @@ -0,0 +1,97 @@ +import pytest +from ceph_volume.devices import raw +from mock.mock import patch + + +class TestRaw(object): + + def test_main_spits_help_with_no_arguments(self, capsys): + raw.main.Raw([]).main() + stdout, stderr = capsys.readouterr() + assert 'Manage a single-device OSD on a raw block device.' in stdout + + def test_main_shows_activate_subcommands(self, capsys): + raw.main.Raw([]).main() + stdout, stderr = capsys.readouterr() + assert 'activate ' in stdout + assert 'Discover and prepare' in stdout + + def test_main_shows_prepare_subcommands(self, capsys): + raw.main.Raw([]).main() + stdout, stderr = capsys.readouterr() + assert 'prepare ' in stdout + assert 'Format a raw device' in stdout + + +class TestPrepare(object): + + def test_main_spits_help_with_no_arguments(self, capsys): + raw.prepare.Prepare([]).main() + stdout, stderr = capsys.readouterr() + assert 'Prepare an OSD by assigning an ID and FSID' in stdout + + def test_main_shows_full_help(self, capsys): + with pytest.raises(SystemExit): + raw.prepare.Prepare(argv=['--help']).main() + stdout, stderr = capsys.readouterr() + assert 'a raw device to use for the OSD' in stdout + assert 'Crush device class to assign this OSD to' in stdout + assert 'Use BlueStore backend' in stdout + assert 'Path to bluestore block.db block device' in stdout + assert 'Path to bluestore block.wal block device' in stdout + assert 'Enable device encryption via dm-crypt' in stdout + + @patch('ceph_volume.util.arg_validators.ValidDevice.__call__') + def test_prepare_dmcrypt_no_secret_passed(self, m_valid_device, capsys): + m_valid_device.return_value = '/dev/foo' + with pytest.raises(SystemExit): + raw.prepare.Prepare(argv=['--bluestore', '--data', '/dev/foo', '--dmcrypt']).main() + stdout, stderr = capsys.readouterr() + assert 'CEPH_VOLUME_DMCRYPT_SECRET is not set, you must set' in stderr + + @patch('ceph_volume.util.encryption.luks_open') + @patch('ceph_volume.util.encryption.luks_format') + @patch('ceph_volume.util.disk.lsblk') + def test_prepare_dmcrypt_block(self, m_lsblk, m_luks_format, m_luks_open): + m_lsblk.return_value = {'KNAME': 'foo'} + m_luks_format.return_value = True + m_luks_open.return_value = True + result = raw.prepare.prepare_dmcrypt('foo', '/dev/foo', 'block', '123') + m_luks_open.assert_called_with('foo', '/dev/foo', 'ceph-123-foo-block-dmcrypt') + m_luks_format.assert_called_with('foo', '/dev/foo') + assert result == '/dev/mapper/ceph-123-foo-block-dmcrypt' + + @patch('ceph_volume.util.encryption.luks_open') + @patch('ceph_volume.util.encryption.luks_format') + @patch('ceph_volume.util.disk.lsblk') + def test_prepare_dmcrypt_db(self, m_lsblk, m_luks_format, m_luks_open): + m_lsblk.return_value = {'KNAME': 'foo'} + m_luks_format.return_value = True + m_luks_open.return_value = True + result = raw.prepare.prepare_dmcrypt('foo', '/dev/foo', 'db', '123') + m_luks_open.assert_called_with('foo', '/dev/foo', 'ceph-123-foo-db-dmcrypt') + m_luks_format.assert_called_with('foo', '/dev/foo') + assert result == '/dev/mapper/ceph-123-foo-db-dmcrypt' + + @patch('ceph_volume.util.encryption.luks_open') + @patch('ceph_volume.util.encryption.luks_format') + @patch('ceph_volume.util.disk.lsblk') + def test_prepare_dmcrypt_wal(self, m_lsblk, m_luks_format, m_luks_open): + m_lsblk.return_value = {'KNAME': 'foo'} + m_luks_format.return_value = True + m_luks_open.return_value = True + result = raw.prepare.prepare_dmcrypt('foo', '/dev/foo', 'wal', '123') + m_luks_open.assert_called_with('foo', '/dev/foo', 'ceph-123-foo-wal-dmcrypt') + m_luks_format.assert_called_with('foo', '/dev/foo') + assert result == '/dev/mapper/ceph-123-foo-wal-dmcrypt' + + @patch('ceph_volume.devices.raw.prepare.rollback_osd') + @patch('ceph_volume.devices.raw.prepare.Prepare.prepare') + @patch('ceph_volume.util.arg_validators.ValidDevice.__call__') + def test_safe_prepare_exception_raised(self, m_valid_device, m_prepare, m_rollback_osd): + m_valid_device.return_value = '/dev/foo' + m_prepare.side_effect=Exception('foo') + m_rollback_osd.return_value = 'foobar' + with pytest.raises(Exception): + raw.prepare.Prepare(argv=['--bluestore', '--data', '/dev/foo']).main() + m_rollback_osd.assert_called() diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini index 829a928d1..f7969fe9b 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini @@ -34,7 +34,7 @@ changedir= centos8-bluestore-mixed_type_explicit: {toxinidir}/centos8/bluestore/mixed-type-explicit centos8-bluestore-mixed_type_dmcrypt_explicit: {toxinidir}/centos8/bluestore/mixed-type-dmcrypt-explicit commands= - git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible + git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible python -m pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt # bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:""} {posargs:--provider=virtualbox} diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm index 65dc95233..c333af3e5 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm @@ -11,6 +11,9 @@ osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' copy_admin_key: false +pv_devices: + - /dev/vdb + - /dev/vdc lvm_volumes: - data: data-lv1 data_vg: test_group diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt index 93c870a36..d73637763 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt @@ -12,6 +12,9 @@ osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' copy_admin_key: false +pv_devices: + - /dev/vdb + - /dev/vdc lvm_volumes: - data: data-lv1 data_vg: test_group diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm index cc40419d6..f5f26e7ce 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm @@ -11,6 +11,9 @@ osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' copy_admin_key: false +pv_devices: + - /dev/vdb + - /dev/vdc # test-volume is created by tests/functional/lvm_setup.yml from /dev/sda lvm_volumes: - data: data-lv1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm_dmcrypt b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm_dmcrypt index d09108372..e5c087271 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm_dmcrypt +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/group_vars/filestore_lvm_dmcrypt @@ -12,6 +12,9 @@ osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' copy_admin_key: false +pv_devices: + - /dev/vdb + - /dev/vdc # test-volume is created by tests/functional/lvm_setup.yml from /dev/sda lvm_volumes: - data: data-lv1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml index e8cf077c6..2cf83e477 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml @@ -114,8 +114,8 @@ suffix: sparse register: tmpdir - - name: create a 5GB sparse file - command: fallocate -l 5G {{ tmpdir.path }}/sparse.file + - name: create a 1GB sparse file + command: fallocate -l 1G {{ tmpdir.path }}/sparse.file - name: find an empty loop device command: losetup -f diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml index 9239a3624..42ee40a1b 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml @@ -135,8 +135,8 @@ suffix: sparse register: tmpdir - - name: create a 5GB sparse file - command: fallocate -l 5G {{ tmpdir.path }}/sparse.file + - name: create a 1GB sparse file + command: fallocate -l 1G {{ tmpdir.path }}/sparse.file - name: find an empty loop device command: losetup -f diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini index 1dd7c999f..2b63875bf 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini @@ -32,7 +32,7 @@ changedir= centos8-filestore-prepare_activate: {toxinidir}/xenial/filestore/prepare_activate centos8-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate commands= - git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible + git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py index a8bd07db9..1d99953d5 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py @@ -1,18 +1,33 @@ import pytest +from copy import deepcopy from ceph_volume.util import device from ceph_volume.api import lvm as api class TestDevice(object): - def test_sys_api(self, device_info): + def test_sys_api(self, monkeypatch, device_info): + volume = api.Volume(lv_name='lv', lv_uuid='y', vg_name='vg', + lv_tags={}, lv_path='/dev/VolGroup/lv') + volumes = [] + volumes.append(volume) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: + deepcopy(volumes)) + data = {"/dev/sda": {"foo": "bar"}} device_info(devices=data) disk = device.Device("/dev/sda") assert disk.sys_api assert "foo" in disk.sys_api - def test_lvm_size(self, device_info): + def test_lvm_size(self, monkeypatch, device_info): + volume = api.Volume(lv_name='lv', lv_uuid='y', vg_name='vg', + lv_tags={}, lv_path='/dev/VolGroup/lv') + volumes = [] + volumes.append(volume) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: + deepcopy(volumes)) + # 5GB in size data = {"/dev/sda": {"size": "5368709120"}} device_info(devices=data) @@ -32,12 +47,15 @@ class TestDevice(object): disk = device.Device("vg/lv") assert disk.is_lv - def test_vgs_is_empty(self, device_info, pvolumes, pvolumes_empty, monkeypatch): - BarPVolume = api.PVolume(pv_name='/dev/sda', pv_uuid="0000", pv_tags={}) + def test_vgs_is_empty(self, device_info, monkeypatch): + BarPVolume = api.PVolume(pv_name='/dev/sda', pv_uuid="0000", + pv_tags={}) + pvolumes = [] pvolumes.append(BarPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda populate=True: pvolumes if populate else pvolumes_empty) lsblk = {"TYPE": "disk"} device_info(lsblk=lsblk) + monkeypatch.setattr(api, 'get_pvs', lambda **kwargs: {}) + disk = device.Device("/dev/nvme0n1") assert disk.vgs == [] @@ -50,42 +68,42 @@ class TestDevice(object): disk = device.Device("/dev/nvme0n1") assert len(disk.vgs) == 1 - def test_device_is_device(self, device_info, pvolumes): + def test_device_is_device(self, device_info): data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "device"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.is_device is True - def test_device_is_rotational(self, device_info, pvolumes): + def test_device_is_rotational(self, device_info): data = {"/dev/sda": {"rotational": "1"}} lsblk = {"TYPE": "device"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.rotational - def test_device_is_not_rotational(self, device_info, pvolumes): + def test_device_is_not_rotational(self, device_info): data = {"/dev/sda": {"rotational": "0"}} lsblk = {"TYPE": "device"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert not disk.rotational - def test_device_is_rotational_lsblk(self, device_info, pvolumes): + def test_device_is_rotational_lsblk(self, device_info): data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "device", "ROTA": "1"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.rotational - def test_device_is_not_rotational_lsblk(self, device_info, pvolumes): + def test_device_is_not_rotational_lsblk(self, device_info): data = {"/dev/sda": {"rotational": "0"}} lsblk = {"TYPE": "device", "ROTA": "0"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert not disk.rotational - def test_device_is_rotational_defaults_true(self, device_info, pvolumes): + def test_device_is_rotational_defaults_true(self, device_info): # rotational will default true if no info from sys_api or lsblk is found data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "device", "foo": "bar"} @@ -93,28 +111,35 @@ class TestDevice(object): disk = device.Device("/dev/sda") assert disk.rotational - def test_disk_is_device(self, device_info, pvolumes): + def test_disk_is_device(self, device_info): data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "disk"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.is_device is True - def test_is_partition(self, device_info, pvolumes): + def test_is_partition(self, device_info): data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "part"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.is_partition - def test_is_not_lvm_memeber(self, device_info, pvolumes): + def test_is_not_acceptable_device(self, device_info): + data = {"/dev/dm-0": {"foo": "bar"}} + lsblk = {"TYPE": "mpath"} + device_info(devices=data, lsblk=lsblk) + disk = device.Device("/dev/dm-0") + assert not disk.is_device + + def test_is_not_lvm_memeber(self, device_info): data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "part"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda") assert not disk.is_lvm_member - def test_is_lvm_memeber(self, device_info, pvolumes): + def test_is_lvm_memeber(self, device_info): data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "part"} device_info(devices=data, lsblk=lsblk) @@ -137,15 +162,13 @@ class TestDevice(object): assert not disk.is_mapper @pytest.mark.usefixtures("lsblk_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_is_ceph_disk_lsblk(self, monkeypatch, patch_bluestore_label): disk = device.Device("/dev/sda") assert disk.is_ceph_disk_member @pytest.mark.usefixtures("blkid_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_is_ceph_disk_blkid(self, monkeypatch, patch_bluestore_label): monkeypatch.setattr("ceph_volume.util.device.disk.lsblk", lambda path: {'PARTLABEL': ""}) @@ -153,8 +176,7 @@ class TestDevice(object): assert disk.is_ceph_disk_member @pytest.mark.usefixtures("lsblk_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_is_ceph_disk_member_not_available_lsblk(self, monkeypatch, patch_bluestore_label): disk = device.Device("/dev/sda") assert disk.is_ceph_disk_member @@ -162,8 +184,7 @@ class TestDevice(object): assert "Used by ceph-disk" in disk.rejected_reasons @pytest.mark.usefixtures("blkid_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_is_ceph_disk_member_not_available_blkid(self, monkeypatch, patch_bluestore_label): monkeypatch.setattr("ceph_volume.util.device.disk.lsblk", lambda path: {'PARTLABEL': ""}) @@ -209,7 +230,6 @@ class TestDevice(object): assert "Has BlueStore device label" in disk.rejected_reasons @pytest.mark.usefixtures("device_info_not_ceph_disk_member", - "disable_lvm_queries", "disable_kernel_queries") def test_is_not_ceph_disk_member_lsblk(self, patch_bluestore_label): disk = device.Device("/dev/sda") @@ -254,13 +274,24 @@ class TestDevice(object): assert not disk.available_raw @pytest.mark.parametrize("ceph_type", ["data", "block"]) - def test_used_by_ceph(self, device_info, pvolumes, pvolumes_empty, monkeypatch, ceph_type): - FooPVolume = api.PVolume(pv_name='/dev/sda', pv_uuid="0000", lv_uuid="0000", pv_tags={}, vg_name="vg") - pvolumes.append(FooPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda populate=True: pvolumes if populate else pvolumes_empty) + def test_used_by_ceph(self, device_info, + monkeypatch, ceph_type): data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "part"} - lv_data = {"lv_path": "vg/lv", "vg_name": "vg", "lv_uuid": "0000", "tags": {"ceph.osd_id": 0, "ceph.type": ceph_type}} + FooPVolume = api.PVolume(pv_name='/dev/sda', pv_uuid="0000", + lv_uuid="0000", pv_tags={}, vg_name="vg") + pvolumes = [] + pvolumes.append(FooPVolume) + lv_data = {"lv_name": "lv", "lv_path": "vg/lv", "vg_name": "vg", + "lv_uuid": "0000", "lv_tags": + "ceph.osd_id=0,ceph.type="+ceph_type} + volumes = [] + lv = api.Volume(**lv_data) + volumes.append(lv) + monkeypatch.setattr(api, 'get_pvs', lambda **kwargs: pvolumes) + monkeypatch.setattr(api, 'get_lvs', lambda **kwargs: + deepcopy(volumes)) + device_info(devices=data, lsblk=lsblk, lv=lv_data) vg = api.VolumeGroup(vg_name='foo/bar', vg_free_count=6, vg_extent_size=1073741824) @@ -268,13 +299,15 @@ class TestDevice(object): disk = device.Device("/dev/sda") assert disk.used_by_ceph - def test_not_used_by_ceph(self, device_info, pvolumes, pvolumes_empty, monkeypatch): + def test_not_used_by_ceph(self, device_info, monkeypatch): FooPVolume = api.PVolume(pv_name='/dev/sda', pv_uuid="0000", lv_uuid="0000", pv_tags={}, vg_name="vg") + pvolumes = [] pvolumes.append(FooPVolume) - monkeypatch.setattr(api, 'PVolumes', lambda populate=True: pvolumes if populate else pvolumes_empty) data = {"/dev/sda": {"foo": "bar"}} lsblk = {"TYPE": "part"} lv_data = {"lv_path": "vg/lv", "vg_name": "vg", "lv_uuid": "0000", "tags": {"ceph.osd_id": 0, "ceph.type": "journal"}} + monkeypatch.setattr(api, 'get_pvs', lambda **kwargs: pvolumes) + device_info(devices=data, lsblk=lsblk, lv=lv_data) disk = device.Device("/dev/sda") assert not disk.used_by_ceph @@ -289,33 +322,33 @@ class TestDevice(object): class TestDeviceEncryption(object): - def test_partition_is_not_encrypted_lsblk(self, device_info, pvolumes): + def test_partition_is_not_encrypted_lsblk(self, device_info): lsblk = {'TYPE': 'part', 'FSTYPE': 'xfs'} device_info(lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.is_encrypted is False - def test_partition_is_encrypted_lsblk(self, device_info, pvolumes): + def test_partition_is_encrypted_lsblk(self, device_info): lsblk = {'TYPE': 'part', 'FSTYPE': 'crypto_LUKS'} device_info(lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.is_encrypted is True - def test_partition_is_not_encrypted_blkid(self, device_info, pvolumes): + def test_partition_is_not_encrypted_blkid(self, device_info): lsblk = {'TYPE': 'part'} blkid = {'TYPE': 'ceph data'} device_info(lsblk=lsblk, blkid=blkid) disk = device.Device("/dev/sda") assert disk.is_encrypted is False - def test_partition_is_encrypted_blkid(self, device_info, pvolumes): + def test_partition_is_encrypted_blkid(self, device_info): lsblk = {'TYPE': 'part'} blkid = {'TYPE': 'crypto_LUKS'} device_info(lsblk=lsblk, blkid=blkid) disk = device.Device("/dev/sda") assert disk.is_encrypted is True - def test_mapper_is_encrypted_luks1(self, device_info, pvolumes, pvolumes_empty, monkeypatch): + def test_mapper_is_encrypted_luks1(self, device_info, monkeypatch): status = {'type': 'LUKS1'} monkeypatch.setattr(device, 'encryption_status', lambda x: status) lsblk = {'FSTYPE': 'xfs', 'TYPE': 'lvm'} @@ -324,7 +357,7 @@ class TestDeviceEncryption(object): disk = device.Device("/dev/mapper/uuid") assert disk.is_encrypted is True - def test_mapper_is_encrypted_luks2(self, device_info, pvolumes, pvolumes_empty, monkeypatch): + def test_mapper_is_encrypted_luks2(self, device_info, monkeypatch): status = {'type': 'LUKS2'} monkeypatch.setattr(device, 'encryption_status', lambda x: status) lsblk = {'FSTYPE': 'xfs', 'TYPE': 'lvm'} @@ -333,7 +366,7 @@ class TestDeviceEncryption(object): disk = device.Device("/dev/mapper/uuid") assert disk.is_encrypted is True - def test_mapper_is_encrypted_plain(self, device_info, pvolumes, pvolumes_empty, monkeypatch): + def test_mapper_is_encrypted_plain(self, device_info, monkeypatch): status = {'type': 'PLAIN'} monkeypatch.setattr(device, 'encryption_status', lambda x: status) lsblk = {'FSTYPE': 'xfs', 'TYPE': 'lvm'} @@ -342,7 +375,7 @@ class TestDeviceEncryption(object): disk = device.Device("/dev/mapper/uuid") assert disk.is_encrypted is True - def test_mapper_is_not_encrypted_plain(self, device_info, pvolumes, pvolumes_empty, monkeypatch): + def test_mapper_is_not_encrypted_plain(self, device_info, monkeypatch): monkeypatch.setattr(device, 'encryption_status', lambda x: {}) lsblk = {'FSTYPE': 'xfs', 'TYPE': 'lvm'} blkid = {'TYPE': 'mapper'} @@ -350,7 +383,7 @@ class TestDeviceEncryption(object): disk = device.Device("/dev/mapper/uuid") assert disk.is_encrypted is False - def test_lv_is_encrypted_blkid(self, device_info, pvolumes): + def test_lv_is_encrypted_blkid(self, device_info): lsblk = {'TYPE': 'lvm'} blkid = {'TYPE': 'crypto_LUKS'} device_info(lsblk=lsblk, blkid=blkid) @@ -358,7 +391,7 @@ class TestDeviceEncryption(object): disk.lv_api = {} assert disk.is_encrypted is True - def test_lv_is_not_encrypted_blkid(self, factory, device_info, pvolumes): + def test_lv_is_not_encrypted_blkid(self, factory, device_info): lsblk = {'TYPE': 'lvm'} blkid = {'TYPE': 'xfs'} device_info(lsblk=lsblk, blkid=blkid) @@ -366,7 +399,7 @@ class TestDeviceEncryption(object): disk.lv_api = factory(encrypted=None) assert disk.is_encrypted is False - def test_lv_is_encrypted_lsblk(self, device_info, pvolumes): + def test_lv_is_encrypted_lsblk(self, device_info): lsblk = {'FSTYPE': 'crypto_LUKS', 'TYPE': 'lvm'} blkid = {'TYPE': 'mapper'} device_info(lsblk=lsblk, blkid=blkid) @@ -374,7 +407,7 @@ class TestDeviceEncryption(object): disk.lv_api = {} assert disk.is_encrypted is True - def test_lv_is_not_encrypted_lsblk(self, factory, device_info, pvolumes): + def test_lv_is_not_encrypted_lsblk(self, factory, device_info): lsblk = {'FSTYPE': 'xfs', 'TYPE': 'lvm'} blkid = {'TYPE': 'mapper'} device_info(lsblk=lsblk, blkid=blkid) @@ -382,7 +415,7 @@ class TestDeviceEncryption(object): disk.lv_api = factory(encrypted=None) assert disk.is_encrypted is False - def test_lv_is_encrypted_lvm_api(self, factory, device_info, pvolumes): + def test_lv_is_encrypted_lvm_api(self, factory, device_info): lsblk = {'FSTYPE': 'xfs', 'TYPE': 'lvm'} blkid = {'TYPE': 'mapper'} device_info(lsblk=lsblk, blkid=blkid) @@ -390,7 +423,7 @@ class TestDeviceEncryption(object): disk.lv_api = factory(encrypted=True) assert disk.is_encrypted is True - def test_lv_is_not_encrypted_lvm_api(self, factory, device_info, pvolumes): + def test_lv_is_not_encrypted_lvm_api(self, factory, device_info): lsblk = {'FSTYPE': 'xfs', 'TYPE': 'lvm'} blkid = {'TYPE': 'mapper'} device_info(lsblk=lsblk, blkid=blkid) @@ -452,8 +485,7 @@ class TestCephDiskDevice(object): assert disk.partlabel == 'ceph data' @pytest.mark.usefixtures("blkid_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_is_member_blkid(self, monkeypatch, patch_bluestore_label): monkeypatch.setattr("ceph_volume.util.device.disk.lsblk", lambda path: {'PARTLABEL': ""}) @@ -462,8 +494,7 @@ class TestCephDiskDevice(object): assert disk.is_member is True @pytest.mark.usefixtures("lsblk_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_is_member_lsblk(self, patch_bluestore_label): disk = device.CephDiskDevice(device.Device("/dev/sda")) @@ -479,8 +510,7 @@ class TestCephDiskDevice(object): ceph_types = ['data', 'wal', 'db', 'lockbox', 'journal', 'block'] @pytest.mark.usefixtures("blkid_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_type_blkid(self, monkeypatch, device_info, ceph_partlabel): monkeypatch.setattr("ceph_volume.util.device.disk.lsblk", lambda path: {'PARTLABEL': ''}) @@ -490,8 +520,7 @@ class TestCephDiskDevice(object): @pytest.mark.usefixtures("blkid_ceph_disk_member", "lsblk_ceph_disk_member", - "disable_kernel_queries", - "disable_lvm_queries") + "disable_kernel_queries") def test_type_lsblk(self, device_info, ceph_partlabel): disk = device.CephDiskDevice(device.Device("/dev/sda")) diff --git a/ceph/src/ceph-volume/ceph_volume/util/__init__.py b/ceph/src/ceph-volume/ceph_volume/util/__init__.py index 43c9c9d68..1b5afe970 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/__init__.py +++ b/ceph/src/ceph-volume/ceph_volume/util/__init__.py @@ -98,3 +98,11 @@ def prompt_bool(question, input_=None): terminal.error('Valid false responses are: n, no') terminal.error('That response was invalid, please try again') return prompt_bool(question, input_=input_prompt) + +def merge_dict(x, y): + """ + Return two dicts merged + """ + z = x.copy() + z.update(y) + return z \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/util/device.py b/ceph/src/ceph-volume/ceph_volume/util/device.py index 878a584ce..2d00f6da3 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/device.py +++ b/ceph/src/ceph-volume/ceph_volume/util/device.py @@ -130,8 +130,14 @@ class Device(object): self.sys_api = part break - # start with lvm since it can use an absolute or relative path - lv = lvm.get_lv_from_argument(self.path) + # if the path is not absolute, we have 'vg/lv', let's use LV name + # to get the LV. + if self.path[0] == '/': + lv = lvm.get_first_lv(filters={'lv_path': self.path}) + else: + vgname, lvname = self.path.split('/') + lv = lvm.get_first_lv(filters={'lv_name': lvname, + 'vg_name': vgname}) if lv: self.lv_api = lv self.lvs = [lv] @@ -246,7 +252,6 @@ class Device(object): # actually unused (not 100% sure) and can simply be removed self.vg_name = vgs[0] self._is_lvm_member = True - self.lvs.extend(lvm.get_device_lvs(path)) return self._is_lvm_member @@ -393,7 +398,7 @@ class Device(object): ] rejected = [reason for (k, v, reason) in reasons if self.sys_api.get(k, '') == v] - # reject disks small than 5GB + # reject disks smaller than 5GB if int(self.sys_api.get('size', 0)) < 5368709120: rejected.append('Insufficient space (<5GB)') if self.is_ceph_disk_member: @@ -403,10 +408,15 @@ class Device(object): return rejected def _check_lvm_reject_reasons(self): - rejected = self._check_generic_reject_reasons() - available_vgs = [vg for vg in self.vgs if vg.free >= 5368709120] - if self.vgs and not available_vgs: - rejected.append('Insufficient space (<5GB) on vgs') + rejected = [] + if self.vgs: + available_vgs = [vg for vg in self.vgs if vg.free >= 5368709120] + if not available_vgs: + rejected.append('Insufficient space (<5GB) on vgs') + else: + # only check generic if no vgs are present. Vgs might hold lvs and + # that might cause 'locked' to trigger + rejected.extend(self._check_generic_reject_reasons()) return len(rejected) == 0, rejected diff --git a/ceph/src/ceph-volume/setup.py b/ceph/src/ceph-volume/setup.py index 9bd48178c..44a0d0e46 100644 --- a/ceph/src/ceph-volume/setup.py +++ b/ceph/src/ceph-volume/setup.py @@ -1,4 +1,5 @@ from setuptools import setup, find_packages +import os setup( @@ -13,9 +14,13 @@ setup( keywords='ceph volume disk devices lvm', url="https://github.com/ceph/ceph", zip_safe = False, + install_requires='ceph', + dependency_links=[''.join(['file://', os.path.join(os.getcwd(), '../', + 'python-common#egg=ceph-1.0.0')])], tests_require=[ 'pytest >=2.1.3', 'tox', + 'ceph', ], entry_points = dict( console_scripts = [ diff --git a/ceph/src/ceph-volume/shell_tox.ini b/ceph/src/ceph-volume/shell_tox.ini index 5cd4606e4..e5c390271 100644 --- a/ceph/src/ceph-volume/shell_tox.ini +++ b/ceph/src/ceph-volume/shell_tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36 +envlist = py36, py3 skip_missing_interpreters = true [testenv] diff --git a/ceph/src/ceph-volume/tox.ini b/ceph/src/ceph-volume/tox.ini index dc66681d2..6ccf3e775 100644 --- a/ceph/src/ceph-volume/tox.ini +++ b/ceph/src/ceph-volume/tox.ini @@ -1,14 +1,15 @@ [tox] -envlist = py27, py35, py36, flake8 +envlist = py36, py3, py3-flake8 skip_missing_interpreters = true [testenv] deps= pytest mock +install_command=./tox_install_command.sh {opts} {packages} commands=py.test -v {posargs:ceph_volume/tests} --ignore=ceph_volume/tests/functional -[testenv:flake8] +[testenv:py3-flake8] deps=flake8 commands=flake8 {posargs:ceph_volume} diff --git a/ceph/src/ceph-volume/tox_install_command.sh b/ceph/src/ceph-volume/tox_install_command.sh new file mode 100755 index 000000000..79343a4c2 --- /dev/null +++ b/ceph/src/ceph-volume/tox_install_command.sh @@ -0,0 +1,3 @@ +#!/bin/bash +python -m pip install --editable="file://`pwd`/../python-common" +python -m pip install $@ diff --git a/ceph/src/ceph.in b/ceph/src/ceph.in index 13929a4cc..44a32bcd6 100755 --- a/ceph/src/ceph.in +++ b/ceph/src/ceph.in @@ -1041,7 +1041,7 @@ def main(): if parsed_args.help: target = None if len(childargs) >= 2 and childargs[0] == 'tell': - target = childargs[1].split('.') + target = childargs[1].split('.', 1) if not validate_target(target): print('target {0} doesn\'t exist; please pass correct target to tell command (e.g., mon.a, osd.1, mds.a, mgr)'.format(childargs[1]), file=sys.stderr) return 1 @@ -1057,7 +1057,7 @@ def main(): # implement "tell service.id help" if len(childargs) >= 3 and childargs[0] == 'tell' and childargs[2] == 'help': - target = childargs[1].split('.') + target = childargs[1].split('.', 1) if validate_target(target): hdr('Tell %s commands' % target[0]) return do_extended_help(parser, childargs, target, None) diff --git a/ceph/src/ceph_osd.cc b/ceph/src/ceph_osd.cc index afa945dd0..a2b36769b 100644 --- a/ceph/src/ceph_osd.cc +++ b/ceph/src/ceph_osd.cc @@ -573,6 +573,9 @@ flushjournal_out: g_conf().get_val("osd_client_message_size_cap"); boost::scoped_ptr client_byte_throttler( new Throttle(g_ceph_context, "osd_client_bytes", message_size)); + uint64_t message_cap = g_conf().get_val("osd_client_message_cap"); + boost::scoped_ptr client_msg_throttler( + new Throttle(g_ceph_context, "osd_client_messages", message_cap)); // All feature bits 0 - 34 should be present from dumpling v0.67 forward uint64_t osd_required = @@ -583,7 +586,7 @@ flushjournal_out: ms_public->set_default_policy(Messenger::Policy::stateless_registered_server(0)); ms_public->set_policy_throttlers(entity_name_t::TYPE_CLIENT, client_byte_throttler.get(), - nullptr); + client_msg_throttler.get()); ms_public->set_policy(entity_name_t::TYPE_MON, Messenger::Policy::lossy_client(osd_required)); ms_public->set_policy(entity_name_t::TYPE_MGR, @@ -752,6 +755,7 @@ flushjournal_out: delete ms_objecter; client_byte_throttler.reset(); + client_msg_throttler.reset(); // cd on exit, so that gmon.out (if any) goes into a separate directory for each node. char s[20]; diff --git a/ceph/src/cephadm/cephadm b/ceph/src/cephadm/cephadm index e27bc73a6..d6f85f1ad 100755 --- a/ceph/src/cephadm/cephadm +++ b/ceph/src/cephadm/cephadm @@ -2,20 +2,20 @@ DEFAULT_IMAGE='docker.io/ceph/ceph:v15' DEFAULT_IMAGE_IS_MASTER=False -LATEST_STABLE_RELEASE='octopus' -DATA_DIR='/var/lib/ceph' -LOG_DIR='/var/log/ceph' -LOCK_DIR='/run/cephadm' -LOGROTATE_DIR='/etc/logrotate.d' -UNIT_DIR='/etc/systemd/system' -LOG_DIR_MODE=0o770 -DATA_DIR_MODE=0o700 +LATEST_STABLE_RELEASE = 'octopus' +DATA_DIR = '/var/lib/ceph' +LOG_DIR = '/var/log/ceph' +LOCK_DIR = '/run/cephadm' +LOGROTATE_DIR = '/etc/logrotate.d' +UNIT_DIR = '/etc/systemd/system' +LOG_DIR_MODE = 0o770 +DATA_DIR_MODE = 0o700 CONTAINER_PREFERENCE = ['podman', 'docker'] # prefer podman to docker -CUSTOM_PS1=r'[ceph: \u@\h \W]\$ ' -DEFAULT_TIMEOUT=None # in seconds -DEFAULT_RETRY=10 -SHELL_DEFAULT_CONF='/etc/ceph/ceph.conf' -SHELL_DEFAULT_KEYRING='/etc/ceph/ceph.client.admin.keyring' +CUSTOM_PS1 = r'[ceph: \u@\h \W]\$ ' +DEFAULT_TIMEOUT = None # in seconds +DEFAULT_RETRY = 10 +SHELL_DEFAULT_CONF = '/etc/ceph/ceph.conf' +SHELL_DEFAULT_KEYRING = '/etc/ceph/ceph.client.admin.keyring' """ You can invoke cephadm in two ways: @@ -41,10 +41,12 @@ You can invoke cephadm in two ways: import argparse import datetime import fcntl +import ipaddress import json import logging import os import platform +import pwd import random import re import select @@ -57,7 +59,7 @@ import tempfile import time import errno try: - from typing import Dict, List, Tuple, Optional, Union, Any, NoReturn, Callable + from typing import Dict, List, Tuple, Optional, Union, Any, NoReturn, Callable, IO except ImportError: pass import uuid @@ -82,6 +84,9 @@ if sys.version_info >= (3, 0): else: from urllib2 import urlopen, HTTPError +if sys.version_info > (3, 0): + unicode = str + container_path = '' cached_stdin = None @@ -93,20 +98,24 @@ class termcolor: red = '\033[31m' end = '\033[0m' + class Error(Exception): pass + class TimeoutExpired(Error): pass ################################## + class Ceph(object): daemons = ('mon', 'mgr', 'mds', 'osd', 'rgw', 'rbd-mirror', 'crash') ################################## + class Monitoring(object): """Define the configs for the monitoring containers""" @@ -167,6 +176,7 @@ class Monitoring(object): ################################## + class NFSGanesha(object): """Defines a NFS-Ganesha container""" @@ -210,14 +220,6 @@ class NFSGanesha(object): # type: (str, Union[int, str]) -> NFSGanesha return cls(fsid, daemon_id, get_parm(args.config_json), args.image) - @staticmethod - def port_in_use(): - # type () -> None - for (srv, port) in NFSGanesha.port_map.items(): - if port_in_use(port): - msg = 'TCP port {} required for {} is already in use'.format(port, srv) - raise Error(msg) - @staticmethod def get_container_mounts(data_dir): # type: (str) -> Dict[str, str] @@ -325,19 +327,20 @@ class NFSGanesha(object): volume_mounts = self.get_container_mounts(data_dir) envs = self.get_container_envs() - logger.info('Creating RADOS grace for action: %s' % (action)) + logger.info('Creating RADOS grace for action: %s' % action) c = CephContainer( image=self.image, entrypoint=entrypoint, args=args, volume_mounts=volume_mounts, - cname=self.get_container_name(desc='grace-%s' % (action)), + cname=self.get_container_name(desc='grace-%s' % action), envs=envs ) return c ################################## + class CephIscsi(object): """Defines a Ceph-Iscsi container""" @@ -384,6 +387,17 @@ class CephIscsi(object): mounts['/dev/log'] = '/dev/log:z' return mounts + @staticmethod + def get_container_binds(): + # type: () -> List[List[str]] + binds = [] + lib_modules = ['type=bind', + 'source=/lib/modules', + 'destination=/lib/modules', + 'ro=true'] + binds.append(lib_modules) + return binds + @staticmethod def get_version(container_id): # type: (str) -> Optional[str] @@ -392,7 +406,7 @@ class CephIscsi(object): [container_path, 'exec', container_id, '/usr/bin/python3', '-c', "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"]) if code == 0: - version = out + version = out.strip() return version def validate(self): @@ -461,8 +475,18 @@ class CephIscsi(object): "umount {0}; fi".format(mount_path) return cmd.split() + def get_tcmu_runner_container(self): + # type: () -> CephContainer + tcmu_container = get_container(self.fsid, self.daemon_type, self.daemon_id) + tcmu_container.entrypoint = "/usr/bin/tcmu-runner" + tcmu_container.volume_mounts.pop("/dev/log") + tcmu_container.volume_mounts["/dev"] = "/dev:z" + tcmu_container.cname = self.get_container_name(desc='tcmu') + return tcmu_container + ################################## + def get_supported_daemons(): # type: () -> List[str] supported_daemons = list(Ceph.daemons) @@ -474,6 +498,7 @@ def get_supported_daemons(): ################################## + def attempt_bind(s, address, port): # type: (socket.socket, str, int) -> None try: @@ -489,6 +514,7 @@ def attempt_bind(s, address, port): finally: s.close() + def port_in_use(port_num): # type: (int) -> bool """Detect whether a port is in use on the local machine - IPv4 and IPv6""" @@ -504,6 +530,7 @@ def port_in_use(port_num): else: return False + def check_ip_port(ip, port): # type: (str, int) -> None if not args.skip_ping_check: @@ -530,6 +557,7 @@ try: except NameError: TimeoutError = OSError + class Timeout(TimeoutError): """ Raised when the lock could not be acquired in *timeout* @@ -563,7 +591,7 @@ class _Acquire_ReturnProxy(object): class FileLock(object): - def __init__(self, name, timeout = -1): + def __init__(self, name, timeout=-1): if not os.path.exists(LOCK_DIR): os.mkdir(LOCK_DIR, 0o700) self._lock_file = os.path.join(LOCK_DIR, name + '.lock') @@ -643,14 +671,14 @@ class FileLock(object): lock_id, lock_filename, poll_intervall ) time.sleep(poll_intervall) - except: + except: # noqa # Something did go wrong, so decrement the counter. self._lock_counter = max(0, self._lock_counter - 1) raise return _Acquire_ReturnProxy(lock = self) - def release(self, force = False): + def release(self, force=False): """ Releases the file lock. Please note, that the lock is only completly released, if the lock @@ -683,10 +711,9 @@ class FileLock(object): return None def __del__(self): - self.release(force = True) + self.release(force=True) return None - def _acquire(self): open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC fd = os.open(self._lock_file, open_mode) @@ -706,8 +733,8 @@ class FileLock(object): # https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition fd = self._lock_file_fd self._lock_file_fd = None - fcntl.flock(fd, fcntl.LOCK_UN) - os.close(fd) + fcntl.flock(fd, fcntl.LOCK_UN) # type: ignore + os.close(fd) # type: ignore return None @@ -765,9 +792,10 @@ def call(command, # type: List[str] end_time = start_time + timeout while not stop: if end_time and (time.time() >= end_time): - logger.info(desc + ':timeout after %s seconds' % timeout) stop = True - process.kill() + if process.poll() is None: + logger.info(desc + ':timeout after %s seconds' % timeout) + process.kill() if reads and process.poll() is not None: # we want to stop, but first read off anything remaining # on stdout/stderr @@ -814,6 +842,8 @@ def call(command, # type: List[str] assert False except (IOError, OSError): pass + logger.debug(desc + ':profile rt=%s, stop=%s, exit=%s, reads=%s' + % (time.time()-start_time, stop, process.poll(), reads)) returncode = process.wait() @@ -888,6 +918,7 @@ def call_timeout(command, timeout): ################################## + def is_available(what, func): # type: (str, Callable[[], bool]) -> None """ @@ -897,12 +928,12 @@ def is_available(what, func): :param func: the callable object that determines availability """ retry = args.retry - logger.info('Waiting for %s...' % (what)) + logger.info('Waiting for %s...' % what) num = 1 while True: if func(): logger.info('%s is available' - % (what)) + % what) break elif num > retry: raise Error('%s not available after %s tries' @@ -937,11 +968,13 @@ def read_config(fn): return cp + def pathify(p): # type: (str) -> str p = os.path.expanduser(p) return os.path.abspath(p) + def get_file_timestamp(fn): # type: (str) -> Optional[str] try: @@ -952,6 +985,7 @@ def get_file_timestamp(fn): except Exception as e: return None + def try_convert_datetime(s): # type: (str) -> Optional[str] # This is super irritating because @@ -992,6 +1026,7 @@ def try_convert_datetime(s): pass return None + def get_podman_version(): # type: () -> Tuple[int, ...] if 'podman' not in container_path: @@ -999,6 +1034,7 @@ def get_podman_version(): out, _, _ = call_throws([container_path, '--version']) return _parse_podman_version(out) + def _parse_podman_version(out): # type: (str) -> Tuple[int, ...] _, _, version_str = out.strip().split() @@ -1018,24 +1054,29 @@ def get_hostname(): # type: () -> str return socket.gethostname() + def get_fqdn(): # type: () -> str return socket.getfqdn() or socket.gethostname() + def get_arch(): # type: () -> str return platform.uname().machine + def generate_service_id(): # type: () -> str return get_hostname() + '.' + ''.join(random.choice(string.ascii_lowercase) for _ in range(6)) + def generate_password(): # type: () -> str return ''.join(random.choice(string.ascii_lowercase + string.digits) for i in range(10)) + def normalize_container_id(i): # type: (str) -> str # docker adds the sha256: prefix, but AFAICS both @@ -1047,10 +1088,12 @@ def normalize_container_id(i): i = i[len(prefix):] return i + def make_fsid(): # type: () -> str return str(uuid.uuid1()) + def is_fsid(s): # type: (str) -> bool try: @@ -1059,6 +1102,7 @@ def is_fsid(s): return False return True + def infer_fsid(func): """ If we only find a single fsid in /var/lib/ceph/*, use that @@ -1069,14 +1113,19 @@ def infer_fsid(func): logger.debug('Using specified fsid: %s' % args.fsid) return func() - fsids = set() + fsids_set = set() daemon_list = list_daemons(detail=False) for daemon in daemon_list: - if 'name' not in args or not args.name: - fsids.add(daemon['fsid']) + if not is_fsid(daemon['fsid']): + # 'unknown' fsid + continue + elif 'name' not in args or not args.name: + # args.name not specified + fsids_set.add(daemon['fsid']) elif daemon['name'] == args.name: - fsids.add(daemon['fsid']) - fsids = list(fsids) + # args.name is a match + fsids_set.add(daemon['fsid']) + fsids = sorted(fsids_set) if not fsids: # some commands do not always require an fsid @@ -1090,6 +1139,7 @@ def infer_fsid(func): return _infer_fsid + def infer_config(func): """ If we find a MON daemon, use the config from that container @@ -1120,6 +1170,7 @@ def infer_config(func): return _infer_config + def _get_default_image(): if DEFAULT_IMAGE_IS_MASTER: warn = '''This is a development version of cephadm. @@ -1130,6 +1181,7 @@ For information regarding the latest stable release: logger.warning('{}{}{}'.format(termcolor.yellow, line, termcolor.end)) return DEFAULT_IMAGE + def infer_image(func): """ Use the most recent ceph image @@ -1146,6 +1198,7 @@ def infer_image(func): return _infer_image + def default_image(func): @wraps(func) def _default_image(): @@ -1163,6 +1216,7 @@ def default_image(func): return _default_image + def get_last_local_ceph_image(): """ :return: The most recent local ceph image (already pulled) @@ -1179,6 +1233,7 @@ def get_last_local_ceph_image(): return r return None + def write_tmp(s, uid, gid): # type: (str, int, int) -> Any tmp_f = tempfile.NamedTemporaryFile(mode='w', @@ -1189,6 +1244,7 @@ def write_tmp(s, uid, gid): return tmp_f + def makedirs(dir, uid, gid, mode): # type: (str, int, int, int) -> None if not os.path.exists(dir): @@ -1198,14 +1254,17 @@ def makedirs(dir, uid, gid, mode): os.chown(dir, uid, gid) os.chmod(dir, mode) # the above is masked by umask... + def get_data_dir(fsid, t, n): # type: (str, str, Union[int, str]) -> str return os.path.join(args.data_dir, fsid, '%s.%s' % (t, n)) + def get_log_dir(fsid): # type: (str) -> str return os.path.join(args.log_dir, fsid) + def make_data_dir_base(fsid, uid, gid): # type: (str, int, int) -> str data_dir_base = os.path.join(args.data_dir, fsid) @@ -1215,30 +1274,34 @@ def make_data_dir_base(fsid, uid, gid): DATA_DIR_MODE) return data_dir_base + def make_data_dir(fsid, daemon_type, daemon_id, uid=None, gid=None): - # type: (str, str, Union[int, str], int, int) -> str - if not uid or not gid: - (uid, gid) = extract_uid_gid() + # type: (str, str, Union[int, str], Optional[int], Optional[int]) -> str + if uid is None or gid is None: + uid, gid = extract_uid_gid() make_data_dir_base(fsid, uid, gid) data_dir = get_data_dir(fsid, daemon_type, daemon_id) makedirs(data_dir, uid, gid, DATA_DIR_MODE) return data_dir + def make_log_dir(fsid, uid=None, gid=None): - # type: (str, int, int) -> str - if not uid or not gid: - (uid, gid) = extract_uid_gid() + # type: (str, Optional[int], Optional[int]) -> str + if uid is None or gid is None: + uid, gid = extract_uid_gid() log_dir = get_log_dir(fsid) makedirs(log_dir, uid, gid, LOG_DIR_MODE) return log_dir + def make_var_run(fsid, uid, gid): # type: (str, int, int) -> None call_throws(['install', '-d', '-m0770', '-o', str(uid), '-g', str(gid), '/var/run/ceph/%s' % fsid]) + def copy_tree(src, dst, uid=None, gid=None): - # type: (List[str], str, int, int) -> None + # type: (List[str], str, Optional[int], Optional[int]) -> None """ Copy a directory tree from src to dst """ @@ -1263,7 +1326,7 @@ def copy_tree(src, dst, uid=None, gid=None): def copy_files(src, dst, uid=None, gid=None): - # type: (List[str], str, int, int) -> None + # type: (List[str], str, Optional[int], Optional[int]) -> None """ Copy a files from src to dst """ @@ -1281,8 +1344,9 @@ def copy_files(src, dst, uid=None, gid=None): logger.debug('chown %s:%s \'%s\'' % (uid, gid, dst_file)) os.chown(dst_file, uid, gid) + def move_files(src, dst, uid=None, gid=None): - # type: (List[str], str, int, int) -> None + # type: (List[str], str, Optional[int], Optional[int]) -> None """ Move files from src to dst """ @@ -1306,6 +1370,7 @@ def move_files(src, dst, uid=None, gid=None): logger.debug('chown %s:%s \'%s\'' % (uid, gid, dst_file)) os.chown(dst_file, uid, gid) + ## copied from distutils ## def find_executable(executable, path=None): """Tries to find 'executable' in the directories listed in 'path'. @@ -1342,6 +1407,7 @@ def find_executable(executable, path=None): return f return None + def find_program(filename): # type: (str) -> str name = find_executable(filename) @@ -1349,6 +1415,7 @@ def find_program(filename): raise ValueError('%s not found' % filename) return name + def get_unit_name(fsid, daemon_type, daemon_id=None): # type: (str, str, Optional[Union[int, str]]) -> str # accept either name or type + id @@ -1357,6 +1424,7 @@ def get_unit_name(fsid, daemon_type, daemon_id=None): else: return 'ceph-%s@%s' % (fsid, daemon_type) + def get_unit_name_by_daemon_name(fsid, name): daemon = get_daemon_description(fsid, name) try: @@ -1364,6 +1432,7 @@ def get_unit_name_by_daemon_name(fsid, name): except KeyError: raise Error('Failed to get unit name for {}'.format(daemon)) + def check_unit(unit_name): # type: (str) -> Tuple[bool, str, bool] # NOTE: we ignore the exit code here because systemctl outputs @@ -1402,6 +1471,7 @@ def check_unit(unit_name): state = 'unknown' return (enabled, state, installed) + def check_units(units, enabler=None): # type: (List[str], Optional[Packager]) -> bool for u in units: @@ -1415,8 +1485,9 @@ def check_units(units, enabler=None): enabler.enable_service(u) return False + def get_legacy_config_fsid(cluster, legacy_dir=None): - # type: (str, str) -> Optional[str] + # type: (str, Optional[str]) -> Optional[str] config_file = '/etc/ceph/%s.conf' % cluster if legacy_dir is not None: config_file = os.path.abspath(legacy_dir + config_file) @@ -1427,8 +1498,9 @@ def get_legacy_config_fsid(cluster, legacy_dir=None): return config.get('global', 'fsid') return None + def get_legacy_daemon_fsid(cluster, daemon_type, daemon_id, legacy_dir=None): - # type: (str, str, Union[int, str], str) -> Optional[str] + # type: (str, str, Union[int, str], Optional[str]) -> Optional[str] fsid = None if daemon_type == 'osd': try: @@ -1446,6 +1518,7 @@ def get_legacy_daemon_fsid(cluster, daemon_type, daemon_id, legacy_dir=None): fsid = get_legacy_config_fsid(cluster, legacy_dir=legacy_dir) return fsid + def get_daemon_args(fsid, daemon_type, daemon_id): # type: (str, str, Union[int, str]) -> List[str] r = list() # type: List[str] @@ -1471,12 +1544,15 @@ def get_daemon_args(fsid, daemon_type, daemon_id): peers = config.get('peers', list()) # type: ignore for peer in peers: r += ["--cluster.peer={}".format(peer)] + # some alertmanager, by default, look elsewhere for a config + r += ["--config.file=/etc/alertmanager/alertmanager.yml"] elif daemon_type == NFSGanesha.daemon_type: nfs_ganesha = NFSGanesha.init(fsid, daemon_id) r += nfs_ganesha.get_daemon_args() return r + def create_daemon_dirs(fsid, daemon_type, daemon_id, uid, gid, config=None, keyring=None): # type: (str, str, Union[int, str], int, int, Optional[str], Optional[str]) -> None @@ -1521,7 +1597,6 @@ def create_daemon_dirs(fsid, daemon_type, daemon_id, uid, gid, makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755) makedirs(os.path.join(data_dir_root, config_dir, 'data'), uid, gid, 0o755) - # populate the config directory for the component from the config-json for fname in required_files: if 'files' in config: # type: ignore @@ -1543,6 +1618,7 @@ def create_daemon_dirs(fsid, daemon_type, daemon_id, uid, gid, ceph_iscsi = CephIscsi.init(fsid, daemon_id) ceph_iscsi.create_daemon_dirs(data_dir, uid, gid) + def get_parm(option): # type: (str) -> Dict[str, str] @@ -1577,6 +1653,7 @@ def get_parm(option): else: return js + def get_config_and_keyring(): # type: () -> Tuple[Optional[str], Optional[str]] config = None @@ -1597,7 +1674,19 @@ def get_config_and_keyring(): with open(args.keyring, 'r') as f: keyring = f.read() - return (config, keyring) + return config, keyring + + +def get_container_binds(fsid, daemon_type, daemon_id): + # type: (str, str, Union[int, str, None]) -> List[List[str]] + binds = list() + + if daemon_type == CephIscsi.daemon_type: + assert daemon_id + binds.extend(CephIscsi.get_container_binds()) + + return binds + def get_container_mounts(fsid, daemon_type, daemon_id, no_config=False): @@ -1667,7 +1756,7 @@ def get_container_mounts(fsid, daemon_type, daemon_id, mounts[os.path.join(data_dir, 'etc/grafana/provisioning/datasources')] = '/etc/grafana/provisioning/datasources:Z' mounts[os.path.join(data_dir, 'etc/grafana/certs')] = '/etc/grafana/certs:Z' elif daemon_type == 'alertmanager': - mounts[os.path.join(data_dir, 'etc/alertmanager')] = '/alertmanager:Z' + mounts[os.path.join(data_dir, 'etc/alertmanager')] = '/etc/alertmanager:Z' if daemon_type == NFSGanesha.daemon_type: assert daemon_id @@ -1682,6 +1771,7 @@ def get_container_mounts(fsid, daemon_type, daemon_id, return mounts + def get_container(fsid, daemon_type, daemon_id, privileged=False, ptrace=False, @@ -1718,20 +1808,14 @@ def get_container(fsid, daemon_type, daemon_id, entrypoint = '' name = '' - ceph_args = [] # type: List[str] + ceph_args = [] # type: List[str] if daemon_type in Monitoring.components: uid, gid = extract_uid_gid_monitoring(daemon_type) - m = Monitoring.components[daemon_type] # type: ignore - metadata = m.get('image', dict()) # type: ignore monitoring_args = [ '--user', str(uid), # FIXME: disable cpu/memory limits for the time being (not supported # by ubuntu 18.04 kernel!) - #'--cpus', - #metadata.get('cpus', '2'), - #'--memory', - #metadata.get('memory', '4GB') ] container_args.extend(monitoring_args) elif daemon_type == 'crash': @@ -1739,7 +1823,7 @@ def get_container(fsid, daemon_type, daemon_id, elif daemon_type in Ceph.daemons: ceph_args = ['-n', name, '-f'] - envs=[] # type: List[str] + envs = [] # type: List[str] if daemon_type == NFSGanesha.daemon_type: envs.extend(NFSGanesha.get_container_envs()) @@ -1749,31 +1833,50 @@ def get_container(fsid, daemon_type, daemon_id, args=ceph_args + get_daemon_args(fsid, daemon_type, daemon_id), container_args=container_args, volume_mounts=get_container_mounts(fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(fsid, daemon_type, daemon_id), cname='ceph-%s-%s.%s' % (fsid, daemon_type, daemon_id), envs=envs, privileged=privileged, ptrace=ptrace, ) + def extract_uid_gid(img='', file_path='/var/lib/ceph'): - # type: (str, str) -> Tuple[int, int] + # type: (str, Union[str, List[str]]) -> Tuple[int, int] if not img: img = args.image - out = CephContainer( - image=img, - entrypoint='stat', - args=['-c', '%u %g', file_path] - ).run() - (uid, gid) = out.split(' ') - return (int(uid), int(gid)) + if isinstance(file_path, str): + paths = [file_path] + else: + paths = file_path + + for fp in paths: + try: + out = CephContainer( + image=img, + entrypoint='stat', + args=['-c', '%u %g', fp] + ).run() + uid, gid = out.split(' ') + return int(uid), int(gid) + except RuntimeError: + pass + raise RuntimeError('uid/gid not found') + def deploy_daemon(fsid, daemon_type, daemon_id, c, uid, gid, config=None, keyring=None, osd_fsid=None, - reconfig=False): - # type: (str, str, Union[int, str], CephContainer, int, int, Optional[str], Optional[str], Optional[str], Optional[bool]) -> None + reconfig=False, + ports=None): + # type: (str, str, Union[int, str], CephContainer, int, int, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[List[int]]) -> None + + ports = ports or [] + if any([port_in_use(port) for port in ports]): + raise Error("TCP Port(s) '{}' required for {} already in use".format(",".join(map(str, ports)), daemon_type)) + data_dir = get_data_dir(fsid, daemon_type, daemon_id) if reconfig and not os.path.exists(data_dir): raise Error('cannot reconfig, data path %s does not exist' % data_dir) @@ -1836,6 +1939,12 @@ def deploy_daemon(fsid, daemon_type, daemon_id, c, uid, gid, update_firewalld(daemon_type) + # Open ports explicitly required for the daemon + if ports: + fw = Firewalld() + fw.open_ports(ports) + fw.apply_rules() + if reconfig and daemon_type not in Ceph.daemons: # ceph daemons do not need a restart; others (presumably) do to pick # up the new config @@ -1844,6 +1953,21 @@ def deploy_daemon(fsid, daemon_type, daemon_id, c, uid, gid, call_throws(['systemctl', 'restart', get_unit_name(fsid, daemon_type, daemon_id)]) +def _write_container_cmd_to_bash(file_obj, container, comment=None, background=False): + # type: (IO[str], CephContainer, Optional[str], Optional[bool]) -> None + if comment: + # Sometimes adding a comment, espectially if there are multiple containers in one + # unit file, makes it easier to read and grok. + file_obj.write('# ' + comment + '\n') + # Sometimes, adding `--rm` to a run_cmd doesn't work. Let's remove the container manually + file_obj.write('! '+ ' '.join(container.rm_cmd()) + '\n') + # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage` + if 'podman' in container_path: + file_obj.write('! '+ ' '.join(container.rm_cmd(storage=True)) + '\n') + + # container run command + file_obj.write(' '.join(container.run_cmd()) + (' &' if background else '') + '\n') + def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, enable=True, start=True, osd_fsid=None): @@ -1851,28 +1975,34 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, # cmd data_dir = get_data_dir(fsid, daemon_type, daemon_id) with open(data_dir + '/unit.run.new', 'w') as f: + f.write('set -e\n') # pre-start cmd(s) if daemon_type == 'osd': # osds have a pre-start step assert osd_fsid - f.write('# Simple OSDs need chown on startup:\n') - for n in ['block', 'block.db', 'block.wal']: - p = os.path.join(data_dir, n) - f.write('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid)) - f.write('# LVM OSDs use ceph-volume lvm activate:\n') - prestart = CephContainer( - image=args.image, - entrypoint='/usr/sbin/ceph-volume', - args=[ - 'lvm', 'activate', - str(daemon_id), osd_fsid, - '--no-systemd' - ], - privileged=True, - volume_mounts=get_container_mounts(fsid, daemon_type, daemon_id), - cname='ceph-%s-%s.%s-activate' % (fsid, daemon_type, daemon_id), - ) - f.write(' '.join(prestart.run_cmd()) + '\n') + simple_fn = os.path.join('/etc/ceph/osd', + '%s-%s.json.adopted-by-cephadm' % (daemon_id, osd_fsid)) + if os.path.exists(simple_fn): + f.write('# Simple OSDs need chown on startup:\n') + for n in ['block', 'block.db', 'block.wal']: + p = os.path.join(data_dir, n) + f.write('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid)) + else: + f.write('# LVM OSDs use ceph-volume lvm activate:\n') + prestart = CephContainer( + image=args.image, + entrypoint='/usr/sbin/ceph-volume', + args=[ + 'lvm', 'activate', + str(daemon_id), osd_fsid, + '--no-systemd' + ], + privileged=True, + volume_mounts=get_container_mounts(fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(fsid, daemon_type, daemon_id), + cname='ceph-%s-%s.%s-activate' % (fsid, daemon_type, daemon_id), + ) + f.write(' '.join(prestart.run_cmd()) + '\n') elif daemon_type == NFSGanesha.daemon_type: # add nfs to the rados grace db nfs_ganesha = NFSGanesha.init(fsid, daemon_id) @@ -1880,13 +2010,15 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, f.write(' '.join(prestart.run_cmd()) + '\n') elif daemon_type == CephIscsi.daemon_type: f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n') + ceph_iscsi = CephIscsi.init(fsid, daemon_id) + tcmu_container = ceph_iscsi.get_tcmu_runner_container() + _write_container_cmd_to_bash(f, tcmu_container, 'iscsi tcmu-runnter container', background=True) if daemon_type in Ceph.daemons: install_path = find_program('install') f.write('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=fsid, uid=uid, gid=gid)) - # container run command - f.write(' '.join(c.run_cmd()) + '\n') + _write_container_cmd_to_bash(f, c, '%s.%s' % (daemon_type, str(daemon_id))) os.fchmod(f.fileno(), 0o600) os.rename(data_dir + '/unit.run.new', data_dir + '/unit.run') @@ -1904,6 +2036,7 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, ], privileged=True, volume_mounts=get_container_mounts(fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(fsid, daemon_type, daemon_id), cname='ceph-%s-%s.%s-deactivate' % (fsid, daemon_type, daemon_id), ) @@ -1914,6 +2047,10 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, poststop = nfs_ganesha.get_rados_grace_container('remove') f.write(' '.join(poststop.run_cmd()) + '\n') elif daemon_type == CephIscsi.daemon_type: + # make sure we also stop the tcmu container + ceph_iscsi = CephIscsi.init(fsid, daemon_id) + tcmu_container = ceph_iscsi.get_tcmu_runner_container() + f.write('! '+ ' '.join(tcmu_container.stop_cmd()) + '\n') f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n') os.fchmod(f.fileno(), 0o600) os.rename(data_dir + '/unit.poststop.new', @@ -1945,56 +2082,94 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, if start: call_throws(['systemctl', 'start', unit_name]) -def update_firewalld(daemon_type): - # type: (str) -> None - if args.skip_firewalld: - return - cmd = find_executable('firewall-cmd') - if not cmd: - logger.debug('firewalld does not appear to be present') - return - (enabled, state, _) = check_unit('firewalld.service') - if not enabled: - logger.debug('firewalld.service is not enabled') - return - - fw_services = [] - fw_ports = [] - if daemon_type == 'mon': - fw_services.append('ceph-mon') - elif daemon_type in ['mgr', 'mds', 'osd']: - fw_services.append('ceph') - if daemon_type == 'mgr': - fw_ports.append(8080) # dashboard - fw_ports.append(8443) # dashboard - fw_ports.append(9283) # mgr/prometheus exporter - elif daemon_type in Monitoring.port_map.keys(): - fw_ports.extend(Monitoring.port_map[daemon_type]) # prometheus etc - elif daemon_type == NFSGanesha.daemon_type: - fw_services.append('nfs') - for svc in fw_services: - out, err, ret = call([cmd, '--permanent', '--query-service', svc]) + +class Firewalld(object): + def __init__(self): + # type: () -> None + self.available = self.check() + + def check(self): + # type: () -> bool + self.cmd = find_executable('firewall-cmd') + if not self.cmd: + logger.debug('firewalld does not appear to be present') + return False + (enabled, state, _) = check_unit('firewalld.service') + if not enabled: + logger.debug('firewalld.service is not enabled') + return False + if state != "running": + logger.debug('firewalld.service is not running') + return False + + logger.info("firewalld ready") + return True + + def enable_service_for(self, daemon_type): + # type: (str) -> None + if not self.available: + logger.debug('Not possible to enable service <%s>. firewalld.service is not available' % daemon_type) + return + + if daemon_type == 'mon': + svc = 'ceph-mon' + elif daemon_type in ['mgr', 'mds', 'osd']: + svc = 'ceph' + elif daemon_type == NFSGanesha.daemon_type: + svc = 'nfs' + else: + return + + out, err, ret = call([self.cmd, '--permanent', '--query-service', svc], verbose_on_failure=False) if ret: logger.info('Enabling firewalld service %s in current zone...' % svc) - out, err, ret = call([cmd, '--permanent', '--add-service', svc]) + out, err, ret = call([self.cmd, '--permanent', '--add-service', svc]) if ret: raise RuntimeError( 'unable to add service %s to current zone: %s' % (svc, err)) else: logger.debug('firewalld service %s is enabled in current zone' % svc) - for port in fw_ports: - tcp_port = str(port) + '/tcp' - out, err, ret = call([cmd, '--permanent', '--query-port', tcp_port]) - if ret: - logger.info('Enabling firewalld port %s in current zone...' % tcp_port) - out, err, ret = call([cmd, '--permanent', '--add-port', tcp_port]) + + def open_ports(self, fw_ports): + # type: (List[int]) -> None + if not self.available: + logger.debug('Not possible to open ports <%s>. firewalld.service is not available' % fw_ports) + return + + for port in fw_ports: + tcp_port = str(port) + '/tcp' + out, err, ret = call([self.cmd, '--permanent', '--query-port', tcp_port], verbose_on_failure=False) if ret: - raise RuntimeError('unable to add port %s to current zone: %s' % - (tcp_port, err)) - else: - logger.debug('firewalld port %s is enabled in current zone' % tcp_port) - call_throws([cmd, '--reload']) + logger.info('Enabling firewalld port %s in current zone...' % tcp_port) + out, err, ret = call([self.cmd, '--permanent', '--add-port', tcp_port]) + if ret: + raise RuntimeError('unable to add port %s to current zone: %s' % + (tcp_port, err)) + else: + logger.debug('firewalld port %s is enabled in current zone' % tcp_port) + + def apply_rules(self): + # type: () -> None + if not self.available: + return + + call_throws([self.cmd, '--reload']) + + +def update_firewalld(daemon_type): + # type: (str) -> None + firewall = Firewalld() + + firewall.enable_service_for(daemon_type) + + fw_ports = [] + + if daemon_type in Monitoring.port_map.keys(): + fw_ports.extend(Monitoring.port_map[daemon_type]) # prometheus etc + + firewall.open_ports(fw_ports) + firewall.apply_rules() def install_base_units(fsid): # type: (str) -> None @@ -2064,6 +2239,7 @@ def install_base_units(fsid): } """ % fsid) + def get_unit_file(fsid): # type: (str) -> str u = """# generated by cephadm @@ -2106,6 +2282,7 @@ WantedBy=ceph-{fsid}.target ################################## + class CephContainer: def __init__(self, image, @@ -2116,8 +2293,9 @@ class CephContainer: container_args=[], envs=None, privileged=False, - ptrace=False): - # type: (str, str, List[str], Dict[str, str], str, List[str], Optional[List[str]], bool, bool) -> None + ptrace=False, + bind_mounts=None): + # type: (str, str, List[str], Dict[str, str], str, List[str], Optional[List[str]], bool, bool, Optional[List[List[str]]]) -> None self.image = image self.entrypoint = entrypoint self.args = args @@ -2127,17 +2305,19 @@ class CephContainer: self.envs = envs self.privileged = privileged self.ptrace = ptrace + self.bind_mounts = bind_mounts if bind_mounts else [] def run_cmd(self): # type: () -> List[str] - vols = [] # type: List[str] - envs = [] # type: List[str] - cname = [] # type: List[str] - entrypoint = [] # type: List[str] + vols = [] # type: List[str] + envs = [] # type: List[str] + cname = [] # type: List[str] + binds = [] # type: List[str] + entrypoint = [] # type: List[str] if self.entrypoint: entrypoint = ['--entrypoint', self.entrypoint] - priv = [] # type: List[str] + priv = [] # type: List[str] if self.privileged: priv = ['--privileged', # let OSD etc read block devs that haven't been chowned @@ -2147,6 +2327,8 @@ class CephContainer: vols = sum( [['-v', '%s:%s' % (host_dir, container_dir)] for host_dir, container_dir in self.volume_mounts.items()], []) + binds = sum([['--mount', '{}'.format(','.join(bind))] + for bind in self.bind_mounts],[]) envs = [ '-e', 'CONTAINER_IMAGE=%s' % self.image, '-e', 'NODE_NAME=%s' % get_hostname(), @@ -2163,22 +2345,25 @@ class CephContainer: '--ipc=host', ] + self.container_args + priv + \ cname + envs + \ - vols + entrypoint + \ + vols + binds + entrypoint + \ [ self.image ] + self.args # type: ignore def shell_cmd(self, cmd): # type: (List[str]) -> List[str] - priv = [] # type: List[str] + priv = [] # type: List[str] if self.privileged: priv = ['--privileged', # let OSD etc read block devs that haven't been chowned '--group-add=disk'] - vols = [] # type: List[str] + vols = [] # type: List[str] vols = sum( [['-v', '%s:%s' % (host_dir, container_dir)] for host_dir, container_dir in self.volume_mounts.items()], []) + binds = [] # type: List[str] + binds = sum([['--mount', '{}'.format(','.join(bind))] + for bind in self.bind_mounts], []) envs = [ '-e', 'CONTAINER_IMAGE=%s' % self.image, '-e', 'NODE_NAME=%s' % get_hostname(), @@ -2195,7 +2380,7 @@ class CephContainer: '--rm', '--net=host', '--ipc=host', - ] + self.container_args + priv + envs + vols + [ + ] + self.container_args + priv + envs + vols + binds + [ '--entrypoint', cmd[0], self.image ] + cmd[1:] @@ -2209,6 +2394,25 @@ class CephContainer: self.cname, ] + cmd + def rm_cmd(self, storage=False): + # type: (bool) -> List[str] + ret = [ + str(container_path), + 'rm', '-f', + ] + if storage: + ret.append('--storage') + ret.append(self.cname) + return ret + + def stop_cmd(self): + # type () -> List[str] + ret = [ + str(container_path), + 'stop', self.cname, + ] + return ret + def run(self, timeout=DEFAULT_TIMEOUT): # type: (Optional[int]) -> str logger.debug(self.run_cmd()) @@ -2218,6 +2422,7 @@ class CephContainer: ################################## + @infer_image def command_version(): # type: () -> int @@ -2227,15 +2432,43 @@ def command_version(): ################################## + @infer_image def command_pull(): # type: () -> int - logger.info('Pulling latest %s...' % args.image) - call_throws([container_path, 'pull', args.image]) + + _pull_image(args.image) return command_inspect_image() + +def _pull_image(image): + # type: (str) -> None + logger.info('Pulling container image %s...' % image) + + ignorelist = [ + "error creating read-write layer with ID", + "net/http: TLS handshake timeout", + "Digest did not match, expected", + ] + + cmd = [container_path, 'pull', image] + cmd_str = ' '.join(cmd) + + for sleep_secs in [1, 4, 25]: + out, err, ret = call(cmd) + if not ret: + return + + if not any(pattern in err for pattern in ignorelist): + raise RuntimeError('Failed command: %s' % cmd_str) + + logger.info('"%s failed transiently. Retrying. waiting %s seconds...' % (cmd_str, sleep_secs)) + time.sleep(sleep_secs) + + raise RuntimeError('Failed command: %s: maximum retries reached' % cmd_str) ################################## + @infer_image def command_inspect_image(): # type: () -> int @@ -2256,6 +2489,23 @@ def command_inspect_image(): ################################## +def unwrap_ipv6(address): + # type: (str) -> str + if address.startswith('[') and address.endswith(']'): + return address[1:-1] + return address + + +def is_ipv6(address): + # type: (str) -> bool + address = unwrap_ipv6(address) + try: + return ipaddress.ip_address(unicode(address)).version == 6 + except ValueError: + logger.warning("Address: {} isn't a valid IP address".format(address)) + return False + + @default_image def command_bootstrap(): # type: () -> int @@ -2291,14 +2541,16 @@ def command_bootstrap(): mon_id = args.mon_id or hostname mgr_id = args.mgr_id or generate_service_id() logging.info('Cluster fsid: %s' % fsid) + ipv6 = False l = FileLock(fsid) l.acquire() # ip r = re.compile(r':(\d+)$') - base_ip = None + base_ip = '' if args.mon_ip: + ipv6 = is_ipv6(args.mon_ip) hasport = r.findall(args.mon_ip) if hasport: port = int(hasport[0]) @@ -2322,6 +2574,7 @@ def command_bootstrap(): if addr_arg[0] != '[' or addr_arg[-1] != ']': raise Error('--mon-addrv value %s must use square backets' % addr_arg) + ipv6 = addr_arg.count('[') > 1 for addr in addr_arg[1:-1].split(','): hasport = r.findall(addr) if not hasport: @@ -2341,7 +2594,8 @@ def command_bootstrap(): # make sure IP is configured locally, and then figure out the # CIDR network for net, ips in list_networks().items(): - if base_ip in ips: + if ipaddress.ip_address(unicode(unwrap_ipv6(base_ip))) in \ + [ipaddress.ip_address(unicode(ip)) for ip in ips]: mon_network = net logger.info('Mon IP %s is in CIDR network %s' % (base_ip, mon_network)) @@ -2361,9 +2615,11 @@ def command_bootstrap(): cp.write(cpf) config = cpf.getvalue() + if args.registry_json or args.registry_url: + command_registry_login() + if not args.skip_pull: - logger.info('Pulling latest %s container...' % args.image) - call_throws([container_path, 'pull', args.image]) + _pull_image(args.image) logger.info('Extracting ceph user uid/gid from container image...') (uid, gid) = extract_uid_gid() @@ -2498,7 +2754,7 @@ def command_bootstrap(): # wait for the service to become available def is_mon_available(): # type: () -> bool - timeout=args.timeout if args.timeout else 30 # seconds + timeout=args.timeout if args.timeout else 60 # seconds out, err, ret = call(c.run_cmd(), desc=c.entrypoint, timeout=timeout) @@ -2535,12 +2791,17 @@ def command_bootstrap(): logger.info('Setting mon public_network...') cli(['config', 'set', 'mon', 'public_network', mon_network]) + if ipv6: + logger.info('Enabling IPv6 (ms_bind_ipv6)') + cli(['config', 'set', 'global', 'ms_bind_ipv6', 'true']) + # create mgr logger.info('Creating mgr...') mgr_keyring = '[mgr.%s]\n\tkey = %s\n' % (mgr_id, mgr_key) mgr_c = get_container(fsid, 'mgr', mgr_id) + # Note:the default port used by the Prometheus node exporter is opened in fw deploy_daemon(fsid, 'mgr', mgr_id, mgr_c, uid, gid, - config=config, keyring=mgr_keyring) + config=config, keyring=mgr_keyring, ports=[9283]) # output files with open(args.output_keyring, 'w') as f: @@ -2557,7 +2818,7 @@ def command_bootstrap(): logger.info('Waiting for mgr to start...') def is_mgr_available(): # type: () -> bool - timeout=args.timeout if args.timeout else 30 # seconds + timeout=args.timeout if args.timeout else 60 # seconds try: out = cli(['status', '-f', 'json-pretty'], timeout=timeout) j = json.loads(out) @@ -2588,6 +2849,8 @@ def command_bootstrap(): # ssh if not args.skip_ssh: + cli(['config-key', 'set', 'mgr/cephadm/ssh_user', args.ssh_user]) + logger.info('Enabling cephadm module...') cli(['mgr', 'module', 'enable', 'cephadm']) wait_for_mgr_restart() @@ -2619,11 +2882,21 @@ def command_bootstrap(): f.write(ssh_pub) logger.info('Wrote public SSH key to to %s' % args.output_pub_ssh_key) - logger.info('Adding key to root@localhost\'s authorized_keys...') - if not os.path.exists('/root/.ssh'): - os.mkdir('/root/.ssh', 0o700) - auth_keys_file = '/root/.ssh/authorized_keys' + logger.info('Adding key to %s@localhost\'s authorized_keys...' % args.ssh_user) + try: + s_pwd = pwd.getpwnam(args.ssh_user) + except KeyError as e: + raise Error('Cannot find uid/gid for ssh-user: %s' % (args.ssh_user)) + ssh_uid = s_pwd.pw_uid + ssh_gid = s_pwd.pw_gid + ssh_dir = os.path.join(s_pwd.pw_dir, '.ssh') + + if not os.path.exists(ssh_dir): + makedirs(ssh_dir, ssh_uid, ssh_gid, 0o700) + + auth_keys_file = '%s/authorized_keys' % ssh_dir add_newline = False + if os.path.exists(auth_keys_file): with open(auth_keys_file, 'r') as f: f.seek(0, os.SEEK_END) @@ -2631,7 +2904,9 @@ def command_bootstrap(): f.seek(f.tell()-1, os.SEEK_SET) # go to last char if f.read() != '\n': add_newline = True + with open(auth_keys_file, 'a') as f: + os.fchown(f.fileno(), ssh_uid, ssh_gid) # just in case we created it os.fchmod(f.fileno(), 0o600) # just in case we created it if add_newline: f.write('\n') @@ -2639,7 +2914,10 @@ def command_bootstrap(): host = get_hostname() logger.info('Adding host %s...' % host) - cli(['orch', 'host', 'add', host]) + try: + cli(['orch', 'host', 'add', host]) + except RuntimeError as e: + raise Error('Failed to add host <%s>: %s' % (host, e)) if not args.orphan_initial_daemons: for t in ['mon', 'mgr', 'crash']: @@ -2653,7 +2931,17 @@ def command_bootstrap(): logger.info('Deploying %s service with default placement...' % t) cli(['orch', 'apply', t]) + if args.registry_url and args.registry_username and args.registry_password: + cli(['config', 'set', 'mgr', 'mgr/cephadm/registry_url', args.registry_url, '--force']) + cli(['config', 'set', 'mgr', 'mgr/cephadm/registry_username', args.registry_username, '--force']) + cli(['config', 'set', 'mgr', 'mgr/cephadm/registry_password', args.registry_password, '--force']) + if not args.skip_dashboard: + # Configure SSL port (cephadm only allows to configure dashboard SSL port) + # if the user does not want to use SSL he can change this setting once the cluster is up + cli(["config", "set", "mgr", "mgr/dashboard/ssl_server_port" , str(args.ssl_dashboard_port)]) + + # configuring dashboard parameters logger.info('Enabling the dashboard module...') cli(['mgr', 'module', 'enable', 'dashboard']) wait_for_mgr_restart() @@ -2681,6 +2969,11 @@ def command_bootstrap(): out = cli(['config', 'get', 'mgr', 'mgr/dashboard/ssl_server_port']) port = int(out) + # Open dashboard port + fw = Firewalld() + fw.open_ports([port]) + fw.apply_rules() + logger.info('Ceph Dashboard is now available at:\n\n' '\t URL: https://%s:%s/\n' '\t User: %s\n' @@ -2688,7 +2981,7 @@ def command_bootstrap(): get_fqdn(), port, args.initial_dashboard_user, password)) - + if args.apply_spec: logger.info('Applying %s to cluster' % args.apply_spec) @@ -2703,7 +2996,7 @@ def command_bootstrap(): ssh_key = '/etc/ceph/ceph.pub' if args.ssh_public_key: ssh_key = args.ssh_public_key.name - out, err, code = call_throws(['ssh-copy-id', '-f', '-i', ssh_key, 'root@%s' % split[1]]) + out, err, code = call_throws(['ssh-copy-id', '-f', '-i', ssh_key, '%s@%s' % (args.ssh_user, split[1])]) mounts = {} mounts[pathify(args.apply_spec)] = '/tmp/spec.yml:z' @@ -2726,6 +3019,44 @@ def command_bootstrap(): ################################## +def command_registry_login(): + if args.registry_json: + logger.info("Pulling custom registry login info from %s." % args.registry_json) + d = get_parm(args.registry_json) + if d.get('url') and d.get('username') and d.get('password'): + args.registry_url = d.get('url') + args.registry_username = d.get('username') + args.registry_password = d.get('password') + registry_login(args.registry_url, args.registry_username, args.registry_password) + else: + raise Error("json provided for custom registry login did not include all necessary fields. " + "Please setup json file as\n" + "{\n" + " \"url\": \"REGISTRY_URL\",\n" + " \"username\": \"REGISTRY_USERNAME\",\n" + " \"password\": \"REGISTRY_PASSWORD\"\n" + "}\n") + elif args.registry_url and args.registry_username and args.registry_password: + registry_login(args.registry_url, args.registry_username, args.registry_password) + else: + raise Error("Invalid custom registry arguments received. To login to a custom registry include " + "--registry-url, --registry-username and --registry-password " + "options or --registry-json option") + return 0 + +def registry_login(url, username, password): + logger.info("Logging into custom registry.") + try: + out, _, _ = call_throws([container_path, 'login', + '-u', username, + '-p', password, + url]) + except: + raise Error("Failed to login to custom registry @ %s as %s with given password" % (args.registry_url, args.registry_username)) + +################################## + + def extract_uid_gid_monitoring(daemon_type): # type: (str) -> Tuple[int, int] @@ -2736,7 +3067,7 @@ def extract_uid_gid_monitoring(daemon_type): elif daemon_type == 'grafana': uid, gid = extract_uid_gid(file_path='/var/lib/grafana') elif daemon_type == 'alertmanager': - uid, gid = extract_uid_gid(file_path='/etc/alertmanager') + uid, gid = extract_uid_gid(file_path=['/etc/alertmanager', '/etc/prometheus']) else: raise Error("{} not implemented yet".format(daemon_type)) return uid, gid @@ -2766,24 +3097,29 @@ def command_deploy(): else: logger.info('%s daemon %s ...' % ('Deploy', args.name)) + # Get and check ports explicitly required to be opened + daemon_ports = [] # type: List[int] + if args.tcp_ports: + daemon_ports = list(map(int, args.tcp_ports.split())) + if daemon_type in Ceph.daemons: config, keyring = get_config_and_keyring() uid, gid = extract_uid_gid() make_var_run(args.fsid, uid, gid) + c = get_container(args.fsid, daemon_type, daemon_id, ptrace=args.allow_ptrace) deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, osd_fsid=args.osd_fsid, - reconfig=args.reconfig) + reconfig=args.reconfig, + ports=daemon_ports) elif daemon_type in Monitoring.components: # monitoring daemon - prometheus, grafana, alertmanager, node-exporter # Default Checks if not args.reconfig and not redeploy: - daemon_ports = Monitoring.port_map[daemon_type] # type: List[int] - if any([port_in_use(port) for port in daemon_ports]): - raise Error("TCP Port(s) '{}' required for {} is already in use".format(",".join(map(str, daemon_ports)), daemon_type)) + daemon_ports.extend(Monitoring.port_map[daemon_type]) # make sure provided config-json is sufficient config = get_parm(args.config_json) # type: ignore @@ -2801,18 +3137,21 @@ def command_deploy(): uid, gid = extract_uid_gid_monitoring(daemon_type) c = get_container(args.fsid, daemon_type, daemon_id) deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, - reconfig=args.reconfig) + reconfig=args.reconfig, + ports=daemon_ports) elif daemon_type == NFSGanesha.daemon_type: if not args.reconfig and not redeploy: - NFSGanesha.port_in_use() + daemon_ports.extend(NFSGanesha.port_map.values()) + config, keyring = get_config_and_keyring() # TODO: extract ganesha uid/gid (997, 994) ? uid, gid = extract_uid_gid() c = get_container(args.fsid, daemon_type, daemon_id) deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, - reconfig=args.reconfig) + reconfig=args.reconfig, + ports=daemon_ports) elif daemon_type == CephIscsi.daemon_type: config, keyring = get_config_and_keyring() @@ -2820,12 +3159,14 @@ def command_deploy(): c = get_container(args.fsid, daemon_type, daemon_id) deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, - reconfig=args.reconfig) + reconfig=args.reconfig, + ports=daemon_ports) else: raise Error("{} not implemented in command_deploy function".format(daemon_type)) ################################## + @infer_image def command_run(): # type: () -> int @@ -2836,6 +3177,7 @@ def command_run(): ################################## + @infer_fsid @infer_config @infer_image @@ -2865,6 +3207,7 @@ def command_shell(): container_args = [] # type: List[str] mounts = get_container_mounts(args.fsid, daemon_type, daemon_id, no_config=True if args.config else False) + binds = get_container_binds(args.fsid, daemon_type, daemon_id) if args.config: mounts[pathify(args.config)] = '/etc/ceph/ceph.conf:z' if args.keyring: @@ -2900,6 +3243,7 @@ def command_shell(): args=[], container_args=container_args, volume_mounts=mounts, + bind_mounts=binds, envs=args.env, privileged=True) command = c.shell_cmd(command) @@ -2908,6 +3252,7 @@ def command_shell(): ################################## + @infer_fsid def command_enter(): # type: () -> int @@ -2935,6 +3280,7 @@ def command_enter(): ################################## + @infer_fsid @infer_image def command_ceph_volume(): @@ -2977,6 +3323,7 @@ def command_ceph_volume(): ################################## + @infer_fsid def command_unit(): # type: () -> None @@ -2992,6 +3339,7 @@ def command_unit(): ################################## + @infer_fsid def command_logs(): # type: () -> None @@ -3012,6 +3360,7 @@ def command_logs(): ################################## + def list_networks(): # type: () -> Dict[str,List[str]] @@ -3021,10 +3370,17 @@ def list_networks(): #j = json.loads(out) #for x in j: + res = _list_ipv4_networks() + res.update(_list_ipv6_networks()) + return res + + +def _list_ipv4_networks(): out, _, _ = call_throws([find_executable('ip'), 'route', 'ls']) - return _parse_ip_route(out) + return _parse_ipv4_route(out) -def _parse_ip_route(out): + +def _parse_ipv4_route(out): r = {} # type: Dict[str,List[str]] p = re.compile(r'^(\S+) (.*)scope link (.*)src (\S+)') for line in out.splitlines(): @@ -3038,6 +3394,39 @@ def _parse_ip_route(out): r[net].append(ip) return r + +def _list_ipv6_networks(): + routes, _, _ = call_throws([find_executable('ip'), '-6', 'route', 'ls']) + ips, _, _ = call_throws([find_executable('ip'), '-6', 'addr', 'ls']) + return _parse_ipv6_route(routes, ips) + + +def _parse_ipv6_route(routes, ips): + r = {} # type: Dict[str,List[str]] + route_p = re.compile(r'^(\S+) dev (\S+) proto (\S+) metric (\S+) .*pref (\S+)$') + ip_p = re.compile(r'^\s+inet6 (\S+)/(.*)scope (.*)$') + for line in routes.splitlines(): + m = route_p.findall(line) + if not m or m[0][0].lower() == 'default': + continue + net = m[0][0] + if net not in r: + r[net] = [] + + for line in ips.splitlines(): + m = ip_p.findall(line) + if not m: + continue + ip = m[0][0] + # find the network it belongs to + net = [n for n in r.keys() + if ipaddress.ip_address(unicode(ip)) in ipaddress.ip_network(unicode(n))] + if net: + r[net[0]].append(ip) + + return r + + def command_list_networks(): # type: () -> None r = list_networks() @@ -3045,12 +3434,14 @@ def command_list_networks(): ################################## + def command_ls(): # type: () -> None ls = list_daemons(detail=not args.no_detail, legacy_dir=args.legacy_dir) print(json.dumps(ls, indent=4)) + def list_daemons(detail=True, legacy_dir=None): # type: (bool, Optional[str]) -> List[Dict[str, str]] host_version = None @@ -3217,8 +3608,7 @@ def command_adopt(): # type: () -> None if not args.skip_pull: - logger.info('Pulling latest %s container...' % args.image) - call_throws([container_path, 'pull', args.image]) + _pull_image(args.image) (daemon_type, daemon_id) = args.name.split('.', 1) @@ -3479,6 +3869,7 @@ def command_adopt_prometheus(daemon_id, fsid): deploy_daemon(fsid, daemon_type, daemon_id, c, uid, gid) update_firewalld(daemon_type) + def command_adopt_grafana(daemon_id, fsid): # type: (str, str) -> None @@ -3521,7 +3912,6 @@ def command_adopt_grafana(daemon_id, fsid): else: logger.debug("Skipping ssl, missing cert {} or key {}".format(cert, key)) - # data - possible custom dashboards/plugins data_src = '/var/lib/grafana/' data_src = os.path.abspath(args.legacy_dir + data_src) @@ -3533,6 +3923,7 @@ def command_adopt_grafana(daemon_id, fsid): deploy_daemon(fsid, daemon_type, daemon_id, c, uid, gid) update_firewalld(daemon_type) + def command_adopt_alertmanager(daemon_id, fsid): # type: (str, str) -> None @@ -3562,6 +3953,7 @@ def command_adopt_alertmanager(daemon_id, fsid): deploy_daemon(fsid, daemon_type, daemon_id, c, uid, gid) update_firewalld(daemon_type) + def _adjust_grafana_ini(filename): # type: (str) -> None @@ -3637,6 +4029,7 @@ def command_rm_daemon(): ################################## + def command_rm_cluster(): # type: () -> None if not args.force: @@ -3720,13 +4113,16 @@ def check_time_sync(enabler=None): return False return True + def command_check_host(): # type: () -> None + global container_path + errors = [] commands = ['systemctl', 'lvcreate'] if args.docker: - container_path = find_program('docker') + container_path = find_program('docker') else: for i in CONTAINER_PREFERENCE: try: @@ -3764,6 +4160,7 @@ def command_check_host(): ################################## + def command_prepare_host(): # type: () -> None logger.info('Verifying podman|docker is present...') @@ -3799,6 +4196,7 @@ def command_prepare_host(): ################################## + class CustomValidation(argparse.Action): def _check_name(self, values): @@ -3821,6 +4219,7 @@ class CustomValidation(argparse.Action): ################################## + def get_distro(): # type: () -> Tuple[Optional[str], Optional[str], Optional[str]] distro = None @@ -3842,6 +4241,7 @@ def get_distro(): distro_codename = val.lower() return distro, distro_version, distro_codename + class Packager(object): def __init__(self, stable=None, version=None, branch=None, commit=None): assert \ @@ -3969,6 +4369,7 @@ class Apt(Packager): logging.info('Podman did not work. Falling back to docker...') self.install(['docker.io']) + class YumDnf(Packager): DISTRO_NAMES = { 'centos': ('centos', 'el'), @@ -4086,17 +4487,10 @@ class YumDnf(Packager): if self.distro_code.startswith('el'): logger.info('Enabling EPEL...') call_throws([self.tool, 'install', '-y', 'epel-release']) - if self.distro_code == 'el8': - # we also need Ken's copr repo, at least for now - logger.info('Enabling supplementary copr repo ktdreyer/ceph-el8...') - call_throws(['dnf', 'copr', 'enable', '-y', 'ktdreyer/ceph-el8']) def rm_repo(self): if os.path.exists(self.repo_path()): os.unlink(self.repo_path()) - if self.distro_code == 'el8': - logger.info('Disabling supplementary copr repo ktdreyer/ceph-el8...') - call_throws(['dnf', 'copr', 'disable', '-y', 'ktdreyer/ceph-el8']) def install(self, ls): logger.info('Installing packages %s...' % ls) @@ -4234,16 +4628,19 @@ def command_add_repo(): commit=args.dev_commit) pkg.add_repo() + def command_rm_repo(): pkg = create_packager() pkg.rm_repo() + def command_install(): pkg = create_packager() pkg.install(args.packages) ################################## + def _get_parser(): # type: () -> argparse.ArgumentParser parser = argparse.ArgumentParser( @@ -4420,7 +4817,7 @@ def _get_parser(): help='ceph.keyring to pass through to the container') parser_shell.add_argument( '--mount', '-m', - help='file or directory path that will be mounted in container /mnt') + help='mount a file or directory under /mnt in the container') parser_shell.add_argument( '--env', '-e', action='append', @@ -4538,7 +4935,11 @@ def _get_parser(): parser_bootstrap.add_argument( '--initial-dashboard-password', help='Initial password for the initial dashboard user') - + parser_bootstrap.add_argument( + '--ssl-dashboard-port', + type=int, + default = 8443, + help='Port number used to connect with dashboard using SSL') parser_bootstrap.add_argument( '--dashboard-key', type=argparse.FileType('r'), @@ -4560,6 +4961,10 @@ def _get_parser(): '--ssh-public-key', type=argparse.FileType('r'), help='SSH public key') + parser_bootstrap.add_argument( + '--ssh-user', + default='root', + help='set user for SSHing to cluster hosts, passwordless sudo will be needed for non-root users') parser_bootstrap.add_argument( '--skip-mon-network', @@ -4613,12 +5018,24 @@ def _get_parser(): '--apply-spec', help='Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)') - parser_bootstrap.add_argument( '--shared_ceph_folder', metavar='CEPH_SOURCE_FOLDER', help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') + parser_bootstrap.add_argument( + '--registry-url', + help='url for custom registry') + parser_bootstrap.add_argument( + '--registry-username', + help='username for custom registry') + parser_bootstrap.add_argument( + '--registry-password', + help='password for custom registry') + parser_bootstrap.add_argument( + '--registry-json', + help='json file with custom registry login info (URL, Username, Password)') + parser_deploy = subparsers.add_parser( 'deploy', help='deploy a daemon') parser_deploy.set_defaults(func=command_deploy) @@ -4650,6 +5067,9 @@ def _get_parser(): '--skip-firewalld', action='store_true', help='Do not configure firewalld') + parser_deploy.add_argument( + '--tcp-ports', + help='List of tcp ports to open in the host firewall') parser_deploy.add_argument( '--reconfig', action='store_true', @@ -4709,8 +5129,28 @@ def _get_parser(): default=['cephadm'], help='packages') + parser_registry_login = subparsers.add_parser( + 'registry-login', help='log host into authenticated registry') + parser_registry_login.set_defaults(func=command_registry_login) + parser_registry_login.add_argument( + '--registry-url', + help='url for custom registry') + parser_registry_login.add_argument( + '--registry-username', + help='username for custom registry') + parser_registry_login.add_argument( + '--registry-password', + help='password for custom registry') + parser_registry_login.add_argument( + '--registry-json', + help='json file with custom registry login info (URL, Username, Password)') + parser_registry_login.add_argument( + '--fsid', + help='cluster FSID') + return parser + def _parse_args(av): parser = _get_parser() args = parser.parse_args(av) @@ -4718,10 +5158,11 @@ def _parse_args(av): args.command.pop(0) return args + if __name__ == "__main__": # allow argv to be injected try: - av = injected_argv # type: ignore + av = injected_argv # type: ignore except NameError: av = sys.argv[1:] args = _parse_args(av) diff --git a/ceph/src/cephadm/mypy.ini b/ceph/src/cephadm/mypy.ini deleted file mode 100644 index 313abc807..000000000 --- a/ceph/src/cephadm/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -ignore_missing_imports = False diff --git a/ceph/src/cephadm/requirements.txt b/ceph/src/cephadm/requirements.txt new file mode 100644 index 000000000..92716874d --- /dev/null +++ b/ceph/src/cephadm/requirements.txt @@ -0,0 +1 @@ +ipaddress ; python_version < '3.3' diff --git a/ceph/src/cephadm/tests/test_cephadm.py b/ceph/src/cephadm/tests/test_cephadm.py index 785aedac7..ef23a2604 100644 --- a/ceph/src/cephadm/tests/test_cephadm.py +++ b/ceph/src/cephadm/tests/test_cephadm.py @@ -90,5 +90,137 @@ default via 10.3.64.1 dev eno1 proto static metric 100 '192.168.122.0/24': ['192.168.122.1']} ), ]) - def test_parse_ip_route(self, test_input, expected): - assert cd._parse_ip_route(test_input) == expected + def test_parse_ipv4_route(self, test_input, expected): + assert cd._parse_ipv4_route(test_input) == expected + + @pytest.mark.parametrize("test_routes, test_ips, expected", [ + ( +""" +::1 dev lo proto kernel metric 256 pref medium +fdbc:7574:21fe:9200::/64 dev wlp2s0 proto ra metric 600 pref medium +fdd8:591e:4969:6363::/64 dev wlp2s0 proto ra metric 600 pref medium +fde4:8dba:82e1::/64 dev eth1 proto kernel metric 256 expires 1844sec pref medium +fe80::/64 dev tun0 proto kernel metric 256 pref medium +fe80::/64 dev wlp2s0 proto kernel metric 600 pref medium +default dev tun0 proto static metric 50 pref medium +default via fe80::2480:28ec:5097:3fe2 dev wlp2s0 proto ra metric 20600 pref medium +""", +""" +1: lo: mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +2: eth0: mtu 1500 state UP qlen 1000 + inet6 fdd8:591e:4969:6363:4c52:cafe:8dd4:dc4/64 scope global temporary dynamic + valid_lft 86394sec preferred_lft 14394sec + inet6 fdbc:7574:21fe:9200:4c52:cafe:8dd4:dc4/64 scope global temporary dynamic + valid_lft 6745sec preferred_lft 3145sec + inet6 fdd8:591e:4969:6363:103a:abcd:af1f:57f3/64 scope global temporary deprecated dynamic + valid_lft 86394sec preferred_lft 0sec + inet6 fdbc:7574:21fe:9200:103a:abcd:af1f:57f3/64 scope global temporary deprecated dynamic + valid_lft 6745sec preferred_lft 0sec + inet6 fdd8:591e:4969:6363:a128:1234:2bdd:1b6f/64 scope global temporary deprecated dynamic + valid_lft 86394sec preferred_lft 0sec + inet6 fdbc:7574:21fe:9200:a128:1234:2bdd:1b6f/64 scope global temporary deprecated dynamic + valid_lft 6745sec preferred_lft 0sec + inet6 fdd8:591e:4969:6363:d581:4321:380b:3905/64 scope global temporary deprecated dynamic + valid_lft 86394sec preferred_lft 0sec + inet6 fdbc:7574:21fe:9200:d581:4321:380b:3905/64 scope global temporary deprecated dynamic + valid_lft 6745sec preferred_lft 0sec + inet6 fe80::1111:2222:3333:4444/64 scope link noprefixroute + valid_lft forever preferred_lft forever + inet6 fde4:8dba:82e1:0:ec4a:e402:e9df:b357/64 scope global temporary dynamic + valid_lft 1074sec preferred_lft 1074sec + inet6 fde4:8dba:82e1:0:5054:ff:fe72:61af/64 scope global dynamic mngtmpaddr + valid_lft 1074sec preferred_lft 1074sec +12: tun0: mtu 1500 state UNKNOWN qlen 100 + inet6 fe80::cafe:cafe:cafe:cafe/64 scope link stable-privacy + valid_lft forever preferred_lft forever +""", + { + "::1": ["::1"], + "fdbc:7574:21fe:9200::/64": ["fdbc:7574:21fe:9200:4c52:cafe:8dd4:dc4", + "fdbc:7574:21fe:9200:103a:abcd:af1f:57f3", + "fdbc:7574:21fe:9200:a128:1234:2bdd:1b6f", + "fdbc:7574:21fe:9200:d581:4321:380b:3905"], + "fdd8:591e:4969:6363::/64": ["fdd8:591e:4969:6363:4c52:cafe:8dd4:dc4", + "fdd8:591e:4969:6363:103a:abcd:af1f:57f3", + "fdd8:591e:4969:6363:a128:1234:2bdd:1b6f", + "fdd8:591e:4969:6363:d581:4321:380b:3905"], + "fde4:8dba:82e1::/64": ["fde4:8dba:82e1:0:ec4a:e402:e9df:b357", + "fde4:8dba:82e1:0:5054:ff:fe72:61af"], + "fe80::/64": ["fe80::1111:2222:3333:4444", + "fe80::cafe:cafe:cafe:cafe"] + } + )]) + def test_parse_ipv6_route(self, test_routes, test_ips, expected): + assert cd._parse_ipv6_route(test_routes, test_ips) == expected + + def test_is_ipv6(self): + cd.logger = mock.Mock() + for good in ("[::1]", "::1", + "fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"): + assert cd.is_ipv6(good) + for bad in ("127.0.0.1", + "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffg", + "1:2:3:4:5:6:7:8:9", "fd00::1::1", "[fg::1]"): + assert not cd.is_ipv6(bad) + + def test_unwrap_ipv6(self): + def unwrap_test(address, expected): + assert cd.unwrap_ipv6(address) == expected + + tests = [ + ('::1', '::1'), ('[::1]', '::1'), + ('[fde4:8dba:82e1:0:5054:ff:fe6a:357]', 'fde4:8dba:82e1:0:5054:ff:fe6a:357'), + ('can actually be any string', 'can actually be any string'), + ('[but needs to be stripped] ', '[but needs to be stripped] ')] + for address, expected in tests: + unwrap_test(address, expected) + + @mock.patch('cephadm.call_throws') + @mock.patch('cephadm.get_parm') + def test_registry_login(self, get_parm, call_throws): + + # test normal valid login with url, username and password specified + call_throws.return_value = '', '', 0 + args = cd._parse_args(['registry-login', '--registry-url', 'sample-url', '--registry-username', 'sample-user', '--registry-password', 'sample-pass']) + cd.args = args + retval = cd.command_registry_login() + assert retval == 0 + + # test bad login attempt with invalid arguments given + args = cd._parse_args(['registry-login', '--registry-url', 'bad-args-url']) + cd.args = args + with pytest.raises(Exception) as e: + assert cd.command_registry_login() + assert str(e.value) == ('Invalid custom registry arguments received. To login to a custom registry include ' + '--registry-url, --registry-username and --registry-password options or --registry-json option') + + # test normal valid login with json file + get_parm.return_value = {"url": "sample-url", "username": "sample-username", "password": "sample-password"} + args = cd._parse_args(['registry-login', '--registry-json', 'sample-json']) + cd.args = args + retval = cd.command_registry_login() + assert retval == 0 + + # test bad login attempt with bad json file + get_parm.return_value = {"bad-json": "bad-json"} + args = cd._parse_args(['registry-login', '--registry-json', 'sample-json']) + cd.args = args + with pytest.raises(Exception) as e: + assert cd.command_registry_login() + assert str(e.value) == ("json provided for custom registry login did not include all necessary fields. " + "Please setup json file as\n" + "{\n" + " \"url\": \"REGISTRY_URL\",\n" + " \"username\": \"REGISTRY_USERNAME\",\n" + " \"password\": \"REGISTRY_PASSWORD\"\n" + "}\n") + + # test login attempt with valid arguments where login command fails + call_throws.side_effect = Exception + args = cd._parse_args(['registry-login', '--registry-url', 'sample-url', '--registry-username', 'sample-user', '--registry-password', 'sample-pass']) + cd.args = args + with pytest.raises(Exception) as e: + cd.command_registry_login() + assert str(e.value) == "Failed to login to custom registry @ sample-url as sample-user with given password" diff --git a/ceph/src/cephadm/tox.ini b/ceph/src/cephadm/tox.ini index 59d94a4c4..be0ab4ddf 100644 --- a/ceph/src/cephadm/tox.ini +++ b/ceph/src/cephadm/tox.ini @@ -7,9 +7,10 @@ skip_install=true deps = pytest mock + -r{toxinidir}/requirements.txt commands=pytest {posargs} [testenv:mypy] basepython = python3 -deps = mypy -commands = mypy {posargs:cephadm} +deps = mypy==0.782 +commands = mypy --config-file ../mypy.ini {posargs:cephadm} diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc old mode 100644 new mode 100755 index c84da997a..52e818482 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -1735,16 +1735,14 @@ int Client::make_request(MetaRequest *request, // open a session? if (!have_open_session(mds)) { session = _get_or_open_mds_session(mds); - + if (session->state == MetaSession::STATE_REJECTED) { + request->abort(-EPERM); + break; + } // wait if (session->state == MetaSession::STATE_OPENING) { ldout(cct, 10) << "waiting for session to mds." << mds << " to open" << dendl; wait_on_context_list(session->waiting_for_open); - // Abort requests on REJECT from MDS - if (rejected_by_mds.count(mds)) { - request->abort(-EPERM); - break; - } continue; } @@ -2051,20 +2049,6 @@ MetaSession *Client::_open_mds_session(mds_rank_t mds) ceph_assert(em.second); /* not already present */ MetaSession *session = &em.first->second; - // Maybe skip sending a request to open if this MDS daemon - // has previously sent us a REJECT. - if (rejected_by_mds.count(mds)) { - if (rejected_by_mds[mds] == session->addrs) { - ldout(cct, 4) << __func__ << " mds." << mds << " skipping " - "because we were rejected" << dendl; - return session; - } else { - ldout(cct, 4) << __func__ << " mds." << mds << " old inst " - "rejected us, trying with new inst" << dendl; - rejected_by_mds.erase(mds); - } - } - auto m = make_message(CEPH_SESSION_REQUEST_OPEN); m->metadata = metadata; m->supported_features = feature_bitset_t(CEPHFS_FEATURES_CLIENT_SUPPORTED); @@ -2079,16 +2063,21 @@ void Client::_close_mds_session(MetaSession *s) s->con->send_message2(make_message(CEPH_SESSION_REQUEST_CLOSE, s->seq)); } -void Client::_closed_mds_session(MetaSession *s) +void Client::_closed_mds_session(MetaSession *s, int err, bool rejected) { ldout(cct, 5) << __func__ << " mds." << s->mds_num << " seq " << s->seq << dendl; - s->state = MetaSession::STATE_CLOSED; + if (rejected && s->state != MetaSession::STATE_CLOSING) + s->state = MetaSession::STATE_REJECTED; + else + s->state = MetaSession::STATE_CLOSED; s->con->mark_down(); signal_context_list(s->waiting_for_open); mount_cond.notify_all(); - remove_session_caps(s); + remove_session_caps(s, err); kick_requests_closed(s); - mds_sessions.erase(s->mds_num); + mds_ranks_closing.erase(s->mds_num); + if (s->state == MetaSession::STATE_CLOSED) + mds_sessions.erase(s->mds_num); } void Client::handle_client_session(const MConstRef& m) @@ -2110,9 +2099,8 @@ void Client::handle_client_session(const MConstRef& m) if (!missing_features.empty()) { lderr(cct) << "mds." << from << " lacks required features '" << missing_features << "', closing session " << dendl; - rejected_by_mds[session->mds_num] = session->addrs; _close_mds_session(session); - _closed_mds_session(session); + _closed_mds_session(session, -EPERM, true); break; } session->mds_features = std::move(m->supported_features); @@ -2175,8 +2163,7 @@ void Client::handle_client_session(const MConstRef& m) error_str = "unknown error"; lderr(cct) << "mds." << from << " rejected us (" << error_str << ")" << dendl; - rejected_by_mds[session->mds_num] = session->addrs; - _closed_mds_session(session); + _closed_mds_session(session, -EPERM, true); } break; @@ -2204,6 +2191,10 @@ void Client::_kick_stale_sessions() for (auto it = mds_sessions.begin(); it != mds_sessions.end(); ) { MetaSession &s = it->second; + if (s.state == MetaSession::STATE_REJECTED) { + mds_sessions.erase(it++); + continue; + } ++it; if (s.state == MetaSession::STATE_STALE) _closed_mds_session(&s); @@ -3231,8 +3222,10 @@ void Client::put_cap_ref(Inode *in, int cap) } } -int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff) +int Client::get_caps(Fh *fh, int need, int want, int *phave, loff_t endoff) { + Inode *in = fh->inode.get(); + int r = check_pool_perm(in, need); if (r < 0) return r; @@ -3246,6 +3239,12 @@ int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff) return -EBADF; } + if ((fh->mode & CEPH_FILE_MODE_WR) && fh->gen != fd_gen) + return -EBADF; + + if ((in->flags & I_ERROR_FILELOCK) && fh->has_any_filelocks()) + return -EIO; + int implemented; int have = in->caps_issued(&implemented); @@ -4160,7 +4159,7 @@ void Client::remove_all_caps(Inode *in) remove_cap(&in->caps.begin()->second, true); } -void Client::remove_session_caps(MetaSession *s) +void Client::remove_session_caps(MetaSession *s, int err) { ldout(cct, 10) << __func__ << " mds." << s->mds_num << dendl; @@ -4172,7 +4171,10 @@ void Client::remove_session_caps(MetaSession *s) dirty_caps = in->dirty_caps | in->flushing_caps; in->wanted_max_size = 0; in->requested_max_size = 0; + if (in->has_any_filelocks()) + in->flags |= I_ERROR_FILELOCK; } + auto caps = cap->implemented; if (cap->wanted | cap->issued) in->flags |= I_CAP_DROPPED; remove_cap(cap, false); @@ -4187,6 +4189,20 @@ void Client::remove_session_caps(MetaSession *s) in->mark_caps_clean(); put_inode(in.get()); } + caps &= CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_BUFFER; + if (caps && !in->caps_issued_mask(caps, true)) { + if (err == -EBLACKLISTED) { + if (in->oset.dirty_or_tx) { + lderr(cct) << __func__ << " still has dirty data on " << *in << dendl; + in->set_async_err(err); + } + objectcacher->purge_set(&in->oset); + } else { + objectcacher->release_set(&in->oset); + } + _schedule_invalidate_callback(in.get(), 0, 0); + } + signal_cond_list(in->waitfor_caps); } s->flushing_caps_tids.clear(); @@ -6023,18 +6039,39 @@ int Client::mount(const std::string &mount_root, const UserPerm& perms, void Client::_close_sessions() { + for (auto it = mds_sessions.begin(); it != mds_sessions.end(); ) { + if (it->second.state == MetaSession::STATE_REJECTED) + mds_sessions.erase(it++); + else + ++it; + } + while (!mds_sessions.empty()) { // send session closes! for (auto &p : mds_sessions) { if (p.second.state != MetaSession::STATE_CLOSING) { _close_mds_session(&p.second); + mds_ranks_closing.insert(p.first); } } // wait for sessions to close - ldout(cct, 2) << "waiting for " << mds_sessions.size() << " mds sessions to close" << dendl; + double timo = cct->_conf.get_val("client_shutdown_timeout").count(); + ldout(cct, 2) << "waiting for " << mds_ranks_closing.size() << " mds session(s) to close (timeout: " + << timo << "s)" << dendl; std::unique_lock l{client_lock, std::adopt_lock}; - mount_cond.wait(l); + if (!timo) { + mount_cond.wait(l); + } else if (!mount_cond.wait_for(l, ceph::make_timespan(timo), [this] { return mds_ranks_closing.empty(); })) { + ldout(cct, 1) << mds_ranks_closing.size() << " mds(s) did not respond to session close -- timing out." << dendl; + while (!mds_ranks_closing.empty()) { + auto session = mds_sessions.at(*mds_ranks_closing.begin()); + // this prunes entry from mds_sessions and mds_ranks_closing + _closed_mds_session(&session, -ETIMEDOUT); + } + } + + mds_ranks_closing.clear(); l.release(); } } @@ -6084,7 +6121,7 @@ void Client::_abort_mds_sessions(int err) // Force-close all sessions while(!mds_sessions.empty()) { auto& session = mds_sessions.begin()->second; - _closed_mds_session(&session); + _closed_mds_session(&session, err); } } @@ -6309,6 +6346,16 @@ void Client::tick() } trim_cache(true); + + if (blacklisted && mounted && + last_auto_reconnect + 30 * 60 < now && + cct->_conf.get_val("client_reconnect_stale")) { + messenger->client_reset(); + fd_gen++; // invalidate open files + blacklisted = false; + _kick_stale_sessions(); + last_auto_reconnect = now; + } } void Client::renew_caps() @@ -8741,7 +8788,7 @@ int Client::lookup_name(Inode *ino, Inode *parent, const UserPerm& perms) Fh *Client::_create_fh(Inode *in, int flags, int cmode, const UserPerm& perms) { ceph_assert(in); - Fh *f = new Fh(in, flags, cmode, perms); + Fh *f = new Fh(in, flags, cmode, fd_gen, perms); ldout(cct, 10) << __func__ << " " << in->ino << " mode " << cmode << dendl; @@ -8869,7 +8916,8 @@ int Client::_open(Inode *in, int flags, mode_t mode, Fh **fhp, if (cmode & CEPH_FILE_MODE_RD) need |= CEPH_CAP_FILE_RD; - result = get_caps(in, need, want, &have, -1); + Fh fh(in, flags, cmode, fd_gen, perms); + result = get_caps(&fh, need, want, &have, -1); if (result < 0) { ldout(cct, 8) << "Unable to get caps after open of inode " << *in << " . Denying open: " << @@ -9130,7 +9178,7 @@ int Client::uninline_data(Inode *in, Context *onfinish) int Client::read(int fd, char *buf, loff_t size, loff_t offset) { - std::lock_guard lock(client_lock); + std::unique_lock lock(client_lock); tout(cct) << "read" << std::endl; tout(cct) << fd << std::endl; tout(cct) << size << std::endl; @@ -9152,6 +9200,7 @@ int Client::read(int fd, char *buf, loff_t size, loff_t offset) int r = _read(f, offset, size, &bl); ldout(cct, 3) << "read(" << fd << ", " << (void*)buf << ", " << size << ", " << offset << ") = " << r << dendl; if (r >= 0) { + lock.unlock(); bl.begin().copy(bl.length(), buf); r = bl.length(); } @@ -9200,7 +9249,7 @@ retry: want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; else want = CEPH_CAP_FILE_CACHE; - r = get_caps(in, CEPH_CAP_FILE_RD, want, &have, -1); + r = get_caps(f, CEPH_CAP_FILE_RD, want, &have, -1); if (r < 0) { goto done; } @@ -9630,7 +9679,7 @@ int64_t Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf, want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; else want = CEPH_CAP_FILE_BUFFER; - int r = get_caps(in, CEPH_CAP_FILE_WR|CEPH_CAP_AUTH_SHARED, want, &have, endoff); + int r = get_caps(f, CEPH_CAP_FILE_WR|CEPH_CAP_AUTH_SHARED, want, &have, endoff); if (r < 0) return r; @@ -9719,9 +9768,11 @@ int64_t Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf, in->truncate_size, in->truncate_seq, &onfinish); client_lock.unlock(); - onfinish.wait(); + r = onfinish.wait(); client_lock.lock(); _sync_write_commit(in); + if (r < 0) + goto done; } // if we get here, write was successful, update client metadata @@ -10168,6 +10219,9 @@ int Client::_do_filelock(Inode *in, Fh *fh, int lock_type, int op, int sleep, << " type " << fl->l_type << " owner " << owner << " " << fl->l_start << "~" << fl->l_len << dendl; + if (in->flags & I_ERROR_FILELOCK) + return -EIO; + int lock_cmd; if (F_RDLCK == fl->l_type) lock_cmd = CEPH_LOCK_SHARED; @@ -10341,30 +10395,39 @@ void Client::_release_filelocks(Fh *fh) Inode *in = fh->inode.get(); ldout(cct, 10) << __func__ << " " << fh << " ino " << in->ino << dendl; + list activated_locks; + list > to_release; if (fh->fcntl_locks) { auto &lock_state = fh->fcntl_locks; - for(multimap::iterator p = lock_state->held_locks.begin(); - p != lock_state->held_locks.end(); - ++p) - to_release.push_back(pair(CEPH_LOCK_FCNTL, p->second)); + for(auto p = lock_state->held_locks.begin(); p != lock_state->held_locks.end(); ) { + auto q = p++; + if (in->flags & I_ERROR_FILELOCK) { + lock_state->remove_lock(q->second, activated_locks); + } else { + to_release.push_back(pair(CEPH_LOCK_FCNTL, q->second)); + } + } lock_state.reset(); } if (fh->flock_locks) { auto &lock_state = fh->flock_locks; - for(multimap::iterator p = lock_state->held_locks.begin(); - p != lock_state->held_locks.end(); - ++p) - to_release.push_back(pair(CEPH_LOCK_FLOCK, p->second)); + for(auto p = lock_state->held_locks.begin(); p != lock_state->held_locks.end(); ) { + auto q = p++; + if (in->flags & I_ERROR_FILELOCK) { + lock_state->remove_lock(q->second, activated_locks); + } else { + to_release.push_back(pair(CEPH_LOCK_FLOCK, q->second)); + } + } lock_state.reset(); } - if (to_release.empty()) - return; + if ((in->flags & I_ERROR_FILELOCK) && !in->has_any_filelocks()) + in->flags &= ~I_ERROR_FILELOCK; - // mds has already released filelocks if session was closed. - if (in->caps.empty()) + if (to_release.empty()) return; struct flock fl; @@ -10748,6 +10811,7 @@ Inode *Client::open_snapdir(Inode *diri) in->mtime = diri->mtime; in->ctime = diri->ctime; in->btime = diri->btime; + in->atime = diri->atime; in->size = diri->size; in->change_attr = diri->change_attr; @@ -11565,6 +11629,12 @@ int Client::_setxattr(Inode *in, const char *name, const void *value, return -EROFS; } + if (size == 0) { + value = ""; + } else if (value == NULL) { + return -EINVAL; + } + bool posix_acl_xattr = false; if (acl_type == POSIX_ACL) posix_acl_xattr = !strncmp(name, "system.", 7); @@ -11798,7 +11868,8 @@ int Client::ll_removexattr(Inode *in, const char *name, const UserPerm& perms) bool Client::_vxattrcb_quota_exists(Inode *in) { return in->quota.is_enable() && - in->snaprealm && in->snaprealm->ino == in->ino; + (in->snapid != CEPH_NOSNAP || + (in->snaprealm && in->snaprealm->ino == in->ino)); } size_t Client::_vxattrcb_quota(Inode *in, char *val, size_t size) { @@ -13322,7 +13393,10 @@ int Client::ll_read(Fh *fh, loff_t off, loff_t len, bufferlist *bl) /* We can't return bytes written larger than INT_MAX, clamp len to that */ len = std::min(len, (loff_t)INT_MAX); - return _read(fh, off, len, bl); + int r = _read(fh, off, len, bl); + ldout(cct, 3) << "ll_read " << fh << " " << off << "~" << len << " = " << r + << dendl; + return r; } int Client::ll_read_block(Inode *in, uint64_t blockid, @@ -13564,7 +13638,7 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length) } int have; - int r = get_caps(in, CEPH_CAP_FILE_WR, CEPH_CAP_FILE_BUFFER, &have, -1); + int r = get_caps(fh, CEPH_CAP_FILE_WR, CEPH_CAP_FILE_BUFFER, &have, -1); if (r < 0) return r; @@ -14110,8 +14184,7 @@ void Client::ms_handle_remote_reset(Connection *con) case MetaSession::STATE_OPEN: { objecter->maybe_request_map(); /* to check if we are blacklisted */ - const auto& conf = cct->_conf; - if (conf->client_reconnect_stale) { + if (cct->_conf.get_val("client_reconnect_stale")) { ldout(cct, 1) << "reset from mds we were open; close mds session for reconnect" << dendl; _closed_mds_session(s); } else { @@ -14493,14 +14566,14 @@ int Client::start_reclaim(const std::string& uuid, unsigned flags, MetaSession *session; if (!have_open_session(mds)) { session = _get_or_open_mds_session(mds); + if (session->state == MetaSession::STATE_REJECTED) + return -EPERM; if (session->state != MetaSession::STATE_OPENING) { // umounting? return -EINVAL; } ldout(cct, 10) << "waiting for session to mds." << mds << " to open" << dendl; wait_on_context_list(session->waiting_for_open); - if (rejected_by_mds.count(mds)) - return -EPERM; continue; } diff --git a/ceph/src/client/Client.h b/ceph/src/client/Client.h index a927d85b1..1b85421c9 100644 --- a/ceph/src/client/Client.h +++ b/ceph/src/client/Client.h @@ -627,14 +627,14 @@ public: inodeno_t realm, int flags, const UserPerm& perms); void remove_cap(Cap *cap, bool queue_release); void remove_all_caps(Inode *in); - void remove_session_caps(MetaSession *session); + void remove_session_caps(MetaSession *session, int err); int mark_caps_flushing(Inode *in, ceph_tid_t *ptid); void adjust_session_flushing_caps(Inode *in, MetaSession *old_s, MetaSession *new_s); void flush_caps_sync(); void kick_flushing_caps(Inode *in, MetaSession *session); void kick_flushing_caps(MetaSession *session); void early_kick_flushing_caps(MetaSession *session); - int get_caps(Inode *in, int need, int want, int *have, loff_t endoff); + int get_caps(Fh *fh, int need, int want, int *have, loff_t endoff); int get_caps_used(Inode *in); void maybe_update_snaprealm(SnapRealm *realm, snapid_t snap_created, snapid_t snap_highwater, @@ -766,7 +766,7 @@ protected: MetaSession *_get_or_open_mds_session(mds_rank_t mds); MetaSession *_open_mds_session(mds_rank_t mds); void _close_mds_session(MetaSession *s); - void _closed_mds_session(MetaSession *s); + void _closed_mds_session(MetaSession *s, int err=0, bool rejected=false); bool _any_stale_sessions() const; void _kick_stale_sessions(); void handle_client_session(const MConstRef& m); @@ -1212,6 +1212,7 @@ private: // mds sessions map mds_sessions; // mds -> push seq + std::set mds_ranks_closing; // mds ranks currently tearing down sessions std::list waiting_for_mdsmap; // FSMap, for when using mds_command @@ -1232,6 +1233,7 @@ private: ceph::unordered_map fd_map; set ll_unclosed_fh_set; ceph::unordered_set opened_dirs; + uint64_t fd_gen = 1; bool initialized = false; bool mounted = false; @@ -1244,12 +1246,6 @@ private: ino_t last_used_faked_ino; ino_t last_used_faked_root; - // When an MDS has sent us a REJECT, remember that and don't - // contact it again. Remember which inst rejected us, so that - // when we talk to another inst with the same rank we can - // try again. - std::map rejected_by_mds; - int local_osd = -ENXIO; epoch_t local_osd_epoch = 0; @@ -1269,6 +1265,8 @@ private: ceph::unordered_map snap_realms; std::map metadata; + utime_t last_auto_reconnect; + // trace generation ofstream traceout; diff --git a/ceph/src/client/Fh.cc b/ceph/src/client/Fh.cc index a51dca312..62bd26140 100644 --- a/ceph/src/client/Fh.cc +++ b/ceph/src/client/Fh.cc @@ -18,9 +18,9 @@ #include "Fh.h" -Fh::Fh(InodeRef in, int flags, int cmode, const UserPerm &perms) : - inode(in), _ref(1), pos(0), mds(0), mode(cmode), flags(flags), pos_locked(false), - actor_perms(perms), readahead() +Fh::Fh(InodeRef in, int flags, int cmode, uint64_t _gen, const UserPerm &perms) : + inode(in), _ref(1), pos(0), mds(0), mode(cmode), gen(_gen), flags(flags), + pos_locked(false), actor_perms(perms), readahead() { inode->add_fh(this); } diff --git a/ceph/src/client/Fh.h b/ceph/src/client/Fh.h index eae96037e..c3355ba6c 100644 --- a/ceph/src/client/Fh.h +++ b/ceph/src/client/Fh.h @@ -17,6 +17,7 @@ struct Fh { loff_t pos; int mds; // have to talk to mds we opened with (for now) int mode; // the mode i opened the file with + uint64_t gen; int flags; bool pos_locked; // pos is currently in use @@ -30,6 +31,12 @@ struct Fh { std::unique_ptr fcntl_locks; std::unique_ptr flock_locks; + bool has_any_filelocks() { + return + (fcntl_locks && !fcntl_locks->empty()) || + (flock_locks && !flock_locks->empty()); + } + // IO error encountered by any writeback on this Inode while // this Fh existed (i.e. an fsync on another Fh will still show // up as an async_err here because it could have been the same @@ -44,7 +51,7 @@ struct Fh { } Fh() = delete; - Fh(InodeRef in, int flags, int cmode, const UserPerm &perms); + Fh(InodeRef in, int flags, int cmode, uint64_t gen, const UserPerm &perms); ~Fh(); void get() { ++_ref; } diff --git a/ceph/src/client/Inode.h b/ceph/src/client/Inode.h index 42dbddc97..afaf64e05 100644 --- a/ceph/src/client/Inode.h +++ b/ceph/src/client/Inode.h @@ -109,11 +109,12 @@ struct CapSnap { }; // inode flags -#define I_COMPLETE 1 -#define I_DIR_ORDERED 2 -#define I_CAP_DROPPED 4 -#define I_SNAPDIR_OPEN 8 -#define I_KICK_FLUSH 16 +#define I_COMPLETE (1 << 0) +#define I_DIR_ORDERED (1 << 1) +#define I_SNAPDIR_OPEN (1 << 2) +#define I_KICK_FLUSH (1 << 3) +#define I_CAP_DROPPED (1 << 4) +#define I_ERROR_FILELOCK (1 << 5) struct Inode { Client *client; @@ -258,6 +259,12 @@ struct Inode { std::unique_ptr fcntl_locks; std::unique_ptr flock_locks; + bool has_any_filelocks() { + return + (fcntl_locks && !fcntl_locks->empty()) || + (flock_locks && !flock_locks->empty()); + } + list delegations; xlist unsafe_ops; diff --git a/ceph/src/client/MetaSession.h b/ceph/src/client/MetaSession.h index 94881c370..c0901305a 100644 --- a/ceph/src/client/MetaSession.h +++ b/ceph/src/client/MetaSession.h @@ -33,6 +33,7 @@ struct MetaSession { STATE_CLOSING, STATE_CLOSED, STATE_STALE, + STATE_REJECTED, } state; enum { diff --git a/ceph/src/cls/rgw/cls_rgw.cc b/ceph/src/cls/rgw/cls_rgw.cc index bdc3f7843..cba9a2d73 100644 --- a/ceph/src/cls/rgw/cls_rgw.cc +++ b/ceph/src/cls/rgw/cls_rgw.cc @@ -1795,7 +1795,9 @@ static int rgw_bucket_unlink_instance(cls_method_context_t hctx, bufferlist *in, olh.update(next_key, next.is_delete_marker()); olh.update_log(CLS_RGW_OLH_OP_LINK_OLH, op.op_tag, next_key, next.is_delete_marker()); } else { - /* next_key is empty */ + // next_key is empty, but we need to preserve its name in case this entry + // gets resharded, because this key is used for hash placement + next_key.name = dest_key.name; olh.update(next_key, false); olh.update_log(CLS_RGW_OLH_OP_UNLINK_OLH, op.op_tag, next_key, false); olh.set_exists(false); @@ -3660,7 +3662,7 @@ static int rgw_cls_lc_get_entry(cls_method_context_t hctx, bufferlist *in, buffe return -EINVAL; } - rgw_lc_entry_t lc_entry; + cls_rgw_lc_entry lc_entry; int ret = read_omap_entry(hctx, op.marker, &lc_entry); if (ret < 0) return ret; @@ -3686,7 +3688,7 @@ static int rgw_cls_lc_set_entry(cls_method_context_t hctx, bufferlist *in, buffe bufferlist bl; encode(op.entry, bl); - int ret = cls_cxx_map_set_val(hctx, op.entry.first, &bl); + int ret = cls_cxx_map_set_val(hctx, op.entry.bucket, &bl); return ret; } @@ -3702,7 +3704,7 @@ static int rgw_cls_lc_rm_entry(cls_method_context_t hctx, bufferlist *in, buffer return -EINVAL; } - int ret = cls_cxx_map_remove_key(hctx, op.entry.first); + int ret = cls_cxx_map_remove_key(hctx, op.entry.bucket); return ret; } @@ -3725,7 +3727,7 @@ static int rgw_cls_lc_get_next_entry(cls_method_context_t hctx, bufferlist *in, if (ret < 0) return ret; map::iterator it; - pair entry; + cls_rgw_lc_entry entry; if (!vals.empty()) { it=vals.begin(); in_iter = it->second.begin(); @@ -3741,7 +3743,8 @@ static int rgw_cls_lc_get_next_entry(cls_method_context_t hctx, bufferlist *in, return 0; } -static int rgw_cls_lc_list_entries(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +static int rgw_cls_lc_list_entries(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { cls_rgw_lc_list_entries_op op; auto in_iter = in->cbegin(); @@ -3752,24 +3755,34 @@ static int rgw_cls_lc_list_entries(cls_method_context_t hctx, bufferlist *in, bu return -EINVAL; } - cls_rgw_lc_list_entries_ret op_ret; + cls_rgw_lc_list_entries_ret op_ret(op.compat_v); bufferlist::const_iterator iter; map vals; string filter_prefix; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals, &op_ret.is_truncated); + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, + &vals, &op_ret.is_truncated); if (ret < 0) return ret; map::iterator it; - pair entry; - for (it = vals.begin(); it != vals.end(); ++it) { + for (auto it = vals.begin(); it != vals.end(); ++it) { + cls_rgw_lc_entry entry; iter = it->second.cbegin(); try { - decode(entry, iter); + decode(entry, iter); } catch (buffer::error& err) { - CLS_LOG(1, "ERROR: rgw_cls_lc_list_entries(): failed to decode entry\n"); - return -EIO; - } - op_ret.entries.insert(entry); + /* try backward compat */ + pair oe; + try { + iter = it->second.begin(); + decode(oe, iter); + entry = {oe.first, 0 /* start */, uint32_t(oe.second)}; + } catch(buffer::error& err) { + CLS_LOG( + 1, "ERROR: rgw_cls_lc_list_entries(): failed to decode entry\n"); + return -EIO; + } + } + op_ret.entries.push_back(entry); } encode(op_ret, *out); return 0; diff --git a/ceph/src/cls/rgw/cls_rgw_client.cc b/ceph/src/cls/rgw/cls_rgw_client.cc index 65e59047e..182897ab4 100644 --- a/ceph/src/cls/rgw/cls_rgw_client.cc +++ b/ceph/src/cls/rgw/cls_rgw_client.cc @@ -839,7 +839,8 @@ int cls_rgw_lc_put_head(IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& h return r; } -int cls_rgw_lc_get_next_entry(IoCtx& io_ctx, const string& oid, string& marker, pair& entry) +int cls_rgw_lc_get_next_entry(IoCtx& io_ctx, const string& oid, string& marker, + cls_rgw_lc_entry& entry) { bufferlist in, out; cls_rgw_lc_get_next_entry_op call; @@ -861,7 +862,8 @@ int cls_rgw_lc_get_next_entry(IoCtx& io_ctx, const string& oid, string& marker, return r; } -int cls_rgw_lc_rm_entry(IoCtx& io_ctx, const string& oid, const pair& entry) +int cls_rgw_lc_rm_entry(IoCtx& io_ctx, const string& oid, + const cls_rgw_lc_entry& entry) { bufferlist in, out; cls_rgw_lc_rm_entry_op call; @@ -871,7 +873,8 @@ int cls_rgw_lc_rm_entry(IoCtx& io_ctx, const string& oid, const pair& entry) +int cls_rgw_lc_set_entry(IoCtx& io_ctx, const string& oid, + const cls_rgw_lc_entry& entry) { bufferlist in, out; cls_rgw_lc_set_entry_op call; @@ -881,7 +884,8 @@ int cls_rgw_lc_set_entry(IoCtx& io_ctx, const string& oid, const pair& entries) + vector& entries) { bufferlist in, out; cls_rgw_lc_list_entries_op op; @@ -930,9 +934,12 @@ int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid, } catch (buffer::error& err) { return -EIO; } - entries.insert(ret.entries.begin(),ret.entries.end()); - return r; + std::sort(std::begin(ret.entries), std::end(ret.entries), + [](const cls_rgw_lc_entry& a, const cls_rgw_lc_entry& b) + { return a.bucket < b.bucket; }); + entries = std::move(ret.entries); + return r; } void cls_rgw_reshard_add(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry) diff --git a/ceph/src/cls/rgw/cls_rgw_client.h b/ceph/src/cls/rgw/cls_rgw_client.h index bd13710e8..c68703c69 100644 --- a/ceph/src/cls/rgw/cls_rgw_client.h +++ b/ceph/src/cls/rgw/cls_rgw_client.h @@ -607,14 +607,14 @@ int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32 #ifndef CLS_CLIENT_HIDE_IOCTX int cls_rgw_lc_get_head(librados::IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head); int cls_rgw_lc_put_head(librados::IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head); -int cls_rgw_lc_get_next_entry(librados::IoCtx& io_ctx, const string& oid, string& marker, pair& entry); -int cls_rgw_lc_rm_entry(librados::IoCtx& io_ctx, const string& oid, const pair& entry); -int cls_rgw_lc_set_entry(librados::IoCtx& io_ctx, const string& oid, const pair& entry); -int cls_rgw_lc_get_entry(librados::IoCtx& io_ctx, const string& oid, const std::string& marker, rgw_lc_entry_t& entry); +int cls_rgw_lc_get_next_entry(librados::IoCtx& io_ctx, const string& oid, string& marker, cls_rgw_lc_entry& entry); +int cls_rgw_lc_rm_entry(librados::IoCtx& io_ctx, const string& oid, const cls_rgw_lc_entry& entry); +int cls_rgw_lc_set_entry(librados::IoCtx& io_ctx, const string& oid, const cls_rgw_lc_entry& entry); +int cls_rgw_lc_get_entry(librados::IoCtx& io_ctx, const string& oid, const std::string& marker, cls_rgw_lc_entry& entry); int cls_rgw_lc_list(librados::IoCtx& io_ctx, const string& oid, const string& marker, uint32_t max_entries, - map& entries); + vector& entries); #endif /* resharding */ diff --git a/ceph/src/cls/rgw/cls_rgw_ops.h b/ceph/src/cls/rgw/cls_rgw_ops.h index d752118b2..873acf070 100644 --- a/ceph/src/cls/rgw/cls_rgw_ops.h +++ b/ceph/src/cls/rgw/cls_rgw_ops.h @@ -1030,21 +1030,26 @@ struct cls_rgw_lc_get_next_entry_op { }; WRITE_CLASS_ENCODER(cls_rgw_lc_get_next_entry_op) -using rgw_lc_entry_t = std::pair; - struct cls_rgw_lc_get_next_entry_ret { - rgw_lc_entry_t entry; + cls_rgw_lc_entry entry; + cls_rgw_lc_get_next_entry_ret() {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 2, bl); encode(entry, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - decode(entry, bl); + DECODE_START(2, bl); + if (struct_v < 2) { + std::pair oe; + decode(oe, bl); + entry = {oe.first, 0 /* start */, uint32_t(oe.second)}; + } else { + decode(entry, bl); + } DECODE_FINISH(bl); } @@ -1071,9 +1076,11 @@ struct cls_rgw_lc_get_entry_op { WRITE_CLASS_ENCODER(cls_rgw_lc_get_entry_op) struct cls_rgw_lc_get_entry_ret { - rgw_lc_entry_t entry; + cls_rgw_lc_entry entry; + cls_rgw_lc_get_entry_ret() {} - cls_rgw_lc_get_entry_ret(rgw_lc_entry_t&& _entry) : entry(std::move(_entry)) {} + cls_rgw_lc_get_entry_ret(cls_rgw_lc_entry&& _entry) + : entry(std::move(_entry)) {} void encode(bufferlist& bl) const { ENCODE_START(1, 1, bl); @@ -1090,38 +1097,49 @@ struct cls_rgw_lc_get_entry_ret { }; WRITE_CLASS_ENCODER(cls_rgw_lc_get_entry_ret) - struct cls_rgw_lc_rm_entry_op { - rgw_lc_entry_t entry; + cls_rgw_lc_entry entry; cls_rgw_lc_rm_entry_op() {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 2, bl); encode(entry, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - decode(entry, bl); + DECODE_START(2, bl); + if (struct_v < 2) { + std::pair oe; + decode(oe, bl); + entry = {oe.first, 0 /* start */, uint32_t(oe.second)}; + } else { + decode(entry, bl); + } DECODE_FINISH(bl); } }; WRITE_CLASS_ENCODER(cls_rgw_lc_rm_entry_op) struct cls_rgw_lc_set_entry_op { - rgw_lc_entry_t entry; + cls_rgw_lc_entry entry; cls_rgw_lc_set_entry_op() {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 2, bl); encode(entry, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - decode(entry, bl); + DECODE_START(2, bl); + if (struct_v < 2) { + std::pair oe; + decode(oe, bl); + entry = {oe.first, 0 /* start */, uint32_t(oe.second)}; + } else { + decode(entry, bl); + } DECODE_FINISH(bl); } }; @@ -1171,18 +1189,20 @@ WRITE_CLASS_ENCODER(cls_rgw_lc_get_head_ret) struct cls_rgw_lc_list_entries_op { string marker; uint32_t max_entries = 0; + uint8_t compat_v{0}; cls_rgw_lc_list_entries_op() {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(3, 1, bl); encode(marker, bl); encode(max_entries, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); + DECODE_START(3, bl); + compat_v = struct_v; decode(marker, bl); decode(max_entries, bl); DECODE_FINISH(bl); @@ -1192,27 +1212,46 @@ struct cls_rgw_lc_list_entries_op { WRITE_CLASS_ENCODER(cls_rgw_lc_list_entries_op) struct cls_rgw_lc_list_entries_ret { - map entries; + vector entries; bool is_truncated{false}; + uint8_t compat_v; - cls_rgw_lc_list_entries_ret() {} +cls_rgw_lc_list_entries_ret(uint8_t compat_v = 3) + : compat_v(compat_v) {} void encode(bufferlist& bl) const { - ENCODE_START(2, 1, bl); - encode(entries, bl); + ENCODE_START(compat_v, 1, bl); + if (compat_v <= 2) { + map oes; + std::for_each(entries.begin(), entries.end(), + [&oes](const cls_rgw_lc_entry& elt) + {oes.insert({elt.bucket, elt.status});}); + encode(oes, bl); + } else { + encode(entries, bl); + } encode(is_truncated, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(2, bl); - decode(entries, bl); + DECODE_START(3, bl); + compat_v = struct_v; + if (struct_v <= 2) { + map oes; + decode(oes, bl); + std::for_each(oes.begin(), oes.end(), + [this](const std::pair& oe) + {entries.push_back({oe.first, 0 /* start */, + uint32_t(oe.second)});}); + } else { + decode(entries, bl); + } if (struct_v >= 2) { decode(is_truncated, bl); } DECODE_FINISH(bl); } - }; WRITE_CLASS_ENCODER(cls_rgw_lc_list_entries_ret) diff --git a/ceph/src/cls/rgw/cls_rgw_types.cc b/ceph/src/cls/rgw/cls_rgw_types.cc index f820003a2..b4753b473 100644 --- a/ceph/src/cls/rgw/cls_rgw_types.cc +++ b/ceph/src/cls/rgw/cls_rgw_types.cc @@ -327,6 +327,7 @@ bool rgw_cls_bi_entry::get_info(cls_rgw_obj_key *key, { rgw_bucket_dir_entry entry; decode(entry, iter); + account = (account && entry.exists); *key = entry.key; *category = entry.meta.category; accounted_stats->num_entries++; diff --git a/ceph/src/cls/rgw/cls_rgw_types.h b/ceph/src/cls/rgw/cls_rgw_types.h index 0bd197ae8..620811dbc 100644 --- a/ceph/src/cls/rgw/cls_rgw_types.h +++ b/ceph/src/cls/rgw/cls_rgw_types.h @@ -1214,6 +1214,37 @@ struct cls_rgw_lc_obj_head }; WRITE_CLASS_ENCODER(cls_rgw_lc_obj_head) +struct cls_rgw_lc_entry { + std::string bucket; + uint64_t start_time; // if in_progress + uint32_t status; + + cls_rgw_lc_entry() + : start_time(0), status(0) {} + + cls_rgw_lc_entry(const cls_rgw_lc_entry& rhs) = default; + + cls_rgw_lc_entry(const std::string& b, uint64_t t, uint32_t s) + : bucket(b), start_time(t), status(s) {}; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(bucket, bl); + encode(start_time, bl); + encode(status, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(bucket, bl); + decode(start_time, bl); + decode(status, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(cls_rgw_lc_entry); + struct cls_rgw_reshard_entry { ceph::real_time time; diff --git a/ceph/src/cls/rgw_gc/cls_rgw_gc.cc b/ceph/src/cls/rgw_gc/cls_rgw_gc.cc index f45fb2df2..bb3c2ffc1 100644 --- a/ceph/src/cls/rgw_gc/cls_rgw_gc.cc +++ b/ceph/src/cls/rgw_gc/cls_rgw_gc.cc @@ -426,7 +426,7 @@ static int cls_rgw_gc_queue_update_entry(cls_method_context_t hctx, bufferlist * } //end - catch auto xattr_iter = xattr_urgent_data_map.find(op.info.tag); if (xattr_iter != xattr_urgent_data_map.end()) { - it->second = op.info.time; + xattr_iter->second = op.info.time; tag_found = true; //write the updated map back bufferlist bl_map; diff --git a/ceph/src/cls/user/cls_user.cc b/ceph/src/cls/user/cls_user.cc index e80e6e231..64018fa1b 100644 --- a/ceph/src/cls/user/cls_user.cc +++ b/ceph/src/cls/user/cls_user.cc @@ -422,7 +422,75 @@ static int cls_user_reset_stats(cls_method_context_t hctx, CLS_LOG(20, "%s: updating header", __func__); return cls_cxx_map_write_header(hctx, &bl); -} +} /* legacy cls_user_reset_stats */ + +/// A method to reset the user.buckets header stats in accordance to +/// the values seen in the user.buckets omap keys. This is not be +/// equivalent to --sync-stats which also re-calculates the stats for +/// each bucket. +static int cls_user_reset_stats2(cls_method_context_t hctx, + buffer::list *in, buffer::list *out) +{ + cls_user_reset_stats2_op op; + + try { + auto bliter = in->cbegin(); + decode(op, bliter); + } catch (ceph::buffer::error& err) { + CLS_LOG(0, "ERROR: %s failed to decode op", __func__); + return -EINVAL; + } + + cls_user_header header; + string from_index{op.marker}, prefix; + cls_user_reset_stats2_ret ret; + + map keys; + int rc = cls_cxx_map_get_vals(hctx, from_index, prefix, MAX_ENTRIES, + &keys, &ret.truncated); + if (rc < 0) { + CLS_LOG(0, "ERROR: %s failed to retrieve omap key-values", __func__); + return rc; + } + CLS_LOG(20, "%s: read %lu key-values, truncated=%d", + __func__, keys.size(), ret.truncated); + + for (const auto& kv : keys) { + cls_user_bucket_entry e; + try { + auto& bl = kv.second; + auto bliter = bl.cbegin(); + decode(e, bliter); + } catch (ceph::buffer::error& err) { + CLS_LOG(0, "ERROR: %s failed to decode bucket entry for %s", + __func__, kv.first.c_str()); + return -EIO; + } + add_header_stats(&ret.acc_stats, e); + } + + /* try-update marker */ + if(!keys.empty()) + ret.marker = (--keys.cend())->first; + + if (! ret.truncated) { + buffer::list bl; + header.last_stats_update = op.time; + header.stats = ret.acc_stats; + encode(header, bl); + + CLS_LOG(20, "%s: updating header", __func__); + rc = cls_cxx_map_write_header(hctx, &bl); + + /* return final result */ + encode(ret, *out); + return rc; + } + + /* return partial result */ + encode(ret, *out); + return 0; +} /* cls_user_reset_stats2 */ CLS_INIT(user) { @@ -435,6 +503,7 @@ CLS_INIT(user) cls_method_handle_t h_user_list_buckets; cls_method_handle_t h_user_get_header; cls_method_handle_t h_user_reset_stats; + cls_method_handle_t h_user_reset_stats2; cls_register("user", &h_class); @@ -447,6 +516,8 @@ CLS_INIT(user) cls_register_cxx_method(h_class, "list_buckets", CLS_METHOD_RD, cls_user_list_buckets, &h_user_list_buckets); cls_register_cxx_method(h_class, "get_header", CLS_METHOD_RD, cls_user_get_header, &h_user_get_header); cls_register_cxx_method(h_class, "reset_user_stats", CLS_METHOD_RD | CLS_METHOD_WR, cls_user_reset_stats, &h_user_reset_stats); + cls_register_cxx_method(h_class, "reset_user_stats2", CLS_METHOD_RD | CLS_METHOD_WR, cls_user_reset_stats2, &h_user_reset_stats2); + return; } diff --git a/ceph/src/cls/user/cls_user_ops.h b/ceph/src/cls/user/cls_user_ops.h index fa4b1f31b..b64c9aebf 100644 --- a/ceph/src/cls/user/cls_user_ops.h +++ b/ceph/src/cls/user/cls_user_ops.h @@ -156,6 +156,69 @@ struct cls_user_reset_stats_op { }; WRITE_CLASS_ENCODER(cls_user_reset_stats_op); +struct cls_user_reset_stats2_op { + ceph::real_time time; + std::string marker; + cls_user_stats acc_stats; + + cls_user_reset_stats2_op() {} + + void encode(ceph::buffer::list& bl) const { + ENCODE_START(1, 1, bl); + encode(time, bl); + encode(marker, bl); + encode(acc_stats, bl); + ENCODE_FINISH(bl); + } + + void decode(ceph::buffer::list::const_iterator& bl) { + DECODE_START(1, bl); + decode(time, bl); + decode(marker, bl); + decode(acc_stats, bl); + DECODE_FINISH(bl); + } + + void dump(ceph::Formatter *f) const; + static void generate_test_instances(std::list& ls); +}; +WRITE_CLASS_ENCODER(cls_user_reset_stats2_op); + +struct cls_user_reset_stats2_ret { + std::string marker; + cls_user_stats acc_stats; /* 0-initialized */ + bool truncated; + + cls_user_reset_stats2_ret() + : truncated(false) {} + + void update_call(cls_user_reset_stats2_op& call) { + call.marker = marker; + call.acc_stats = acc_stats; + } + + void encode(ceph::buffer::list& bl) const { + ENCODE_START(1, 1, bl); + encode(marker, bl); + encode(acc_stats, bl); + encode(truncated, bl); + ENCODE_FINISH(bl); + } + + void decode(ceph::buffer::list::const_iterator& bl) { + DECODE_START(1, bl); + decode(marker, bl); + decode(acc_stats, bl); + decode(truncated, bl); + DECODE_FINISH(bl); + } + + void dump(ceph::Formatter *f) const; + static void generate_test_instances( + std::list& ls); +}; +WRITE_CLASS_ENCODER(cls_user_reset_stats2_ret); + struct cls_user_get_header_ret { cls_user_header header; diff --git a/ceph/src/common/Preforker.h b/ceph/src/common/Preforker.h index 5951fbeb4..d34179b40 100644 --- a/ceph/src/common/Preforker.h +++ b/ceph/src/common/Preforker.h @@ -3,6 +3,7 @@ #ifndef CEPH_COMMON_PREFORKER_H #define CEPH_COMMON_PREFORKER_H +#include #include #include #include @@ -45,6 +46,17 @@ public: return (errno = e, -1); } + struct sigaction sa; + sa.sa_handler = SIG_IGN; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + if (sigaction(SIGHUP, &sa, nullptr) != 0) { + int e = errno; + oss << "[" << getpid() << "]: unable to ignore SIGHUP: " << cpp_strerror(e); + err = oss.str(); + return (errno = e, -1); + } + forked = true; childpid = fork(); diff --git a/ceph/src/common/legacy_config_opts.h b/ceph/src/common/legacy_config_opts.h index b0562247d..cd7b78f10 100644 --- a/ceph/src/common/legacy_config_opts.h +++ b/ceph/src/common/legacy_config_opts.h @@ -347,7 +347,6 @@ OPTION(client_trace, OPT_STR) OPTION(client_readahead_min, OPT_LONGLONG) // readahead at _least_ this much. OPTION(client_readahead_max_bytes, OPT_LONGLONG) // default unlimited OPTION(client_readahead_max_periods, OPT_LONGLONG) // as multiple of file layout period (object size * num stripes) -OPTION(client_reconnect_stale, OPT_BOOL) // automatically reconnect stale session OPTION(client_snapdir, OPT_STR) OPTION(client_mount_uid, OPT_INT) OPTION(client_mount_gid, OPT_INT) @@ -914,6 +913,8 @@ OPTION(bluefs_sync_write, OPT_BOOL) OPTION(bluefs_allocator, OPT_STR) // stupid | bitmap OPTION(bluefs_preextend_wal_files, OPT_BOOL) // this *requires* that rocksdb has recycling enabled OPTION(bluefs_log_replay_check_allocations, OPT_BOOL) +OPTION(bluefs_replay_recovery, OPT_BOOL) +OPTION(bluefs_replay_recovery_disable_compact, OPT_BOOL) OPTION(bluestore_bluefs, OPT_BOOL) OPTION(bluestore_bluefs_env_mirror, OPT_BOOL) // mirror to normal Env for debug @@ -1276,6 +1277,11 @@ OPTION(rgw_enable_quota_threads, OPT_BOOL) OPTION(rgw_enable_gc_threads, OPT_BOOL) OPTION(rgw_enable_lc_threads, OPT_BOOL) +/* overrides for librgw/nfs */ +OPTION(rgw_nfs_run_gc_threads, OPT_BOOL) +OPTION(rgw_nfs_run_lc_threads, OPT_BOOL) +OPTION(rgw_nfs_run_quota_threads, OPT_BOOL) +OPTION(rgw_nfs_run_sync_thread, OPT_BOOL) OPTION(rgw_data, OPT_STR) OPTION(rgw_enable_apis, OPT_STR) diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index b9d5b675a..2af36dd0e 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -1501,6 +1501,11 @@ std::vector