* [PATCH v3 11/20] docs: kernel_abi.py: Handle with a lazy Sphinx parser
From: Mauro Carvalho Chehab @ 2019-07-17 11:05 UTC (permalink / raw)
To: gregkh; +Cc: Mauro Carvalho Chehab, Jonathan Corbet, linux-doc
In-Reply-To: <cover.1563360659.git.mchehab+samsung@kernel.org>
The Sphinx docutils parser is lazy: if the content is bigger than
a certain number of lines, it silenlty stops parsing it,
producing an incomplete content. This seems to be worse on newer
Sphinx versions, like 2.0.
So, change the logic to parse the contents per input file.
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
---
Documentation/sphinx/kernel_abi.py | 39 ++++++++++++++++++++----------
1 file changed, 26 insertions(+), 13 deletions(-)
diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py
index efa338e18764..a417026ed690 100644
--- a/Documentation/sphinx/kernel_abi.py
+++ b/Documentation/sphinx/kernel_abi.py
@@ -36,6 +36,7 @@ import os
import subprocess
import sys
import re
+import kernellog
from os import path
@@ -79,12 +80,6 @@ class KernelCmd(Directive):
"debug" : directives.flag
}
- def warn(self, message, **replace):
- replace["fname"] = self.state.document.current_source
- replace["line_no"] = replace.get("line_no", self.lineno)
- message = ("%(fname)s:%(line_no)s: [kernel-abi WARN] : " + message) % replace
- self.state.document.settings.env.app.warn(message, prefix="")
-
def run(self):
doc = self.state.document
@@ -110,7 +105,7 @@ class KernelCmd(Directive):
shell_env["srctree"] = srctree
lines = self.runCmd(cmd, shell=True, cwd=cwd, env=shell_env)
- nodeList = self.nestedParse(lines, fname)
+ nodeList = self.nestedParse(lines, self.arguments[0])
return nodeList
def runCmd(self, cmd, **kwargs):
@@ -137,9 +132,9 @@ class KernelCmd(Directive):
% (self.name, ErrorString(exc)))
return out
- def nestedParse(self, lines, f):
+ def nestedParse(self, lines, fname):
content = ViewList()
- node = nodes.section()
+ node = nodes.section()
if "debug" in self.options:
code_block = "\n\n.. code-block:: rst\n :linenos:\n"
@@ -149,22 +144,42 @@ class KernelCmd(Directive):
line_regex = re.compile("^#define LINENO (\S+)\#([0-9]+)$")
ln = 0
+ n = 0
+ f = fname
for line in lines.split("\n"):
+ n = n + 1
match = line_regex.search(line)
if match:
- f = match.group(1)
+ new_f = match.group(1)
+
+ # Sphinx parser is lazy: it stops parsing contents in the
+ # middle, if it is too big. So, handle it per input file
+ if new_f != f and content:
+ self.do_parse(content, node)
+ content = ViewList()
+
+ f = new_f
+
# sphinx counts lines from 0
ln = int(match.group(2)) - 1
else:
content.append(line, f, ln)
- buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
+ kernellog.info(self.state.document.settings.env.app, "%s: parsed %i lines" % (fname, n))
+ if content:
+ self.do_parse(content, node)
+
+ return node.children
+
+ def do_parse(self, content, node):
if Use_SSI:
with switch_source_input(self.state, content):
self.state.nested_parse(content, 0, node, match_titles=1)
else:
+ buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
+
self.state.memo.title_styles = []
self.state.memo.section_level = 0
self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter)
@@ -172,5 +187,3 @@ class KernelCmd(Directive):
self.state.nested_parse(content, 0, node, match_titles=1)
finally:
self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf
-
- return node.children
--
2.21.0
^ permalink raw reply related
* [PATCH v3 09/20] docs: kernel_abi.py: add a SPDX header file
From: Mauro Carvalho Chehab @ 2019-07-17 11:05 UTC (permalink / raw)
To: gregkh; +Cc: Mauro Carvalho Chehab, Jonathan Corbet, linux-doc
In-Reply-To: <cover.1563360659.git.mchehab+samsung@kernel.org>
This file is released under GPL v2.
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
---
Documentation/sphinx/kernel_abi.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py
index ef91b1e1ff4b..5d43cac73d0a 100644
--- a/Documentation/sphinx/kernel_abi.py
+++ b/Documentation/sphinx/kernel_abi.py
@@ -1,4 +1,5 @@
# coding=utf-8
+# SPDX-License-Identifier: GPL-2.0
#
u"""
kernel-abi
--
2.21.0
^ permalink raw reply related
* [PATCH v3 06/20] docs: kernel_abi.py: fix UTF-8 support
From: Mauro Carvalho Chehab @ 2019-07-17 11:05 UTC (permalink / raw)
To: gregkh; +Cc: Mauro Carvalho Chehab, Jonathan Corbet, linux-doc
In-Reply-To: <cover.1563360659.git.mchehab+samsung@kernel.org>
The parser breaks with UTF-8 characters with Sphinx 1.4.
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
---
Documentation/sphinx/kernel_abi.py | 20 +++++++-------------
1 file changed, 7 insertions(+), 13 deletions(-)
diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py
index 32ce90775d96..0f3e51e67e8d 100644
--- a/Documentation/sphinx/kernel_abi.py
+++ b/Documentation/sphinx/kernel_abi.py
@@ -1,4 +1,5 @@
-# -*- coding: utf-8; mode: python -*-
+# coding=utf-8
+#
u"""
kernel-abi
~~~~~~~~~~
@@ -28,6 +29,7 @@ u"""
"""
+import codecs
import sys
import os
from os import path
@@ -43,14 +45,6 @@ from docutils.utils.error_reporting import ErrorString
__version__ = '1.0'
-# We can't assume that six is installed
-PY3 = sys.version_info[0] == 3
-PY2 = sys.version_info[0] == 2
-if PY3:
- # pylint: disable=C0103, W0622
- unicode = str
- basestring = str
-
def setup(app):
app.add_directive("kernel-abi", KernelCmd)
@@ -115,12 +109,12 @@ class KernelCmd(Directive):
cmd
, stdout = subprocess.PIPE
, stderr = subprocess.PIPE
- , universal_newlines = True
, **kwargs
)
out, err = proc.communicate()
- if err:
- self.warn(err)
+
+ out, err = codecs.decode(out, 'utf-8'), codecs.decode(err, 'utf-8')
+
if proc.returncode != 0:
raise self.severe(
u"command '%s' failed with return code %d"
@@ -129,7 +123,7 @@ class KernelCmd(Directive):
except OSError as exc:
raise self.severe(u"problems with '%s' directive: %s."
% (self.name, ErrorString(exc)))
- return unicode(out)
+ return out
def nestedParse(self, lines, fname):
content = ViewList()
--
2.21.0
^ permalink raw reply related
* [PATCH v3 05/20] docs: kernel_abi.py: add a script to parse ABI documentation
From: Mauro Carvalho Chehab @ 2019-07-17 11:05 UTC (permalink / raw)
To: gregkh; +Cc: Mauro Carvalho Chehab, Jonathan Corbet, linux-doc
In-Reply-To: <cover.1563360659.git.mchehab+samsung@kernel.org>
The ABI documentation is special: it is not plain text files,
but, instead, files with an strict format, as specified by
Documentation/ABI/README.
Add a parser for it.
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
---
Documentation/sphinx/kernel_abi.py | 155 +++++++++++++++++++++++++++++
1 file changed, 155 insertions(+)
create mode 100644 Documentation/sphinx/kernel_abi.py
diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py
new file mode 100644
index 000000000000..32ce90775d96
--- /dev/null
+++ b/Documentation/sphinx/kernel_abi.py
@@ -0,0 +1,155 @@
+# -*- coding: utf-8; mode: python -*-
+u"""
+ kernel-abi
+ ~~~~~~~~~~
+
+ Implementation of the ``kernel-abi`` reST-directive.
+
+ :copyright: Copyright (C) 2016 Markus Heiser
+ :copyright: Copyright (C) 2016 Mauro Carvalho Chehab
+ :license: GPL Version 2, June 1991 see Linux/COPYING for details.
+
+ The ``kernel-abi`` (:py:class:`KernelCmd`) directive calls the
+ scripts/get_abi.pl script to parse the Kernel ABI files.
+
+ Overview of directive's argument and options.
+
+ .. code-block:: rst
+
+ .. kernel-abi:: <ABI directory location>
+ :debug:
+
+ The argument ``<ABI directory location>`` is required. It contains the
+ location of the ABI files to be parsed.
+
+ ``debug``
+ Inserts a code-block with the *raw* reST. Sometimes it is helpful to see
+ what reST is generated.
+
+"""
+
+import sys
+import os
+from os import path
+import subprocess
+
+from sphinx.ext.autodoc import AutodocReporter
+
+from docutils import nodes
+from docutils.parsers.rst import Directive, directives
+from docutils.statemachine import ViewList
+from docutils.utils.error_reporting import ErrorString
+
+
+__version__ = '1.0'
+
+# We can't assume that six is installed
+PY3 = sys.version_info[0] == 3
+PY2 = sys.version_info[0] == 2
+if PY3:
+ # pylint: disable=C0103, W0622
+ unicode = str
+ basestring = str
+
+def setup(app):
+
+ app.add_directive("kernel-abi", KernelCmd)
+ return dict(
+ version = __version__
+ , parallel_read_safe = True
+ , parallel_write_safe = True
+ )
+
+class KernelCmd(Directive):
+
+ u"""KernelABI (``kernel-abi``) directive"""
+
+ required_arguments = 1
+ optional_arguments = 0
+ has_content = False
+ final_argument_whitespace = True
+
+ option_spec = {
+ "debug" : directives.flag
+ }
+
+ def warn(self, message, **replace):
+ replace["fname"] = self.state.document.current_source
+ replace["line_no"] = replace.get("line_no", self.lineno)
+ message = ("%(fname)s:%(line_no)s: [kernel-abi WARN] : " + message) % replace
+ self.state.document.settings.env.app.warn(message, prefix="")
+
+ def run(self):
+
+ doc = self.state.document
+ if not doc.settings.file_insertion_enabled:
+ raise self.warning("docutils: file insertion disabled")
+
+ env = doc.settings.env
+ cwd = path.dirname(doc.current_source)
+ cmd = "get_abi.pl rest --dir "
+ cmd += self.arguments[0]
+
+ srctree = path.abspath(os.environ["srctree"])
+
+ fname = cmd
+
+ # extend PATH with $(srctree)/scripts
+ path_env = os.pathsep.join([
+ srctree + os.sep + "scripts",
+ os.environ["PATH"]
+ ])
+ shell_env = os.environ.copy()
+ shell_env["PATH"] = path_env
+ shell_env["srctree"] = srctree
+
+ lines = self.runCmd(cmd, shell=True, cwd=cwd, env=shell_env)
+ nodeList = self.nestedParse(lines, fname)
+ return nodeList
+
+ def runCmd(self, cmd, **kwargs):
+ u"""Run command ``cmd`` and return it's stdout as unicode."""
+
+ try:
+ proc = subprocess.Popen(
+ cmd
+ , stdout = subprocess.PIPE
+ , stderr = subprocess.PIPE
+ , universal_newlines = True
+ , **kwargs
+ )
+ out, err = proc.communicate()
+ if err:
+ self.warn(err)
+ if proc.returncode != 0:
+ raise self.severe(
+ u"command '%s' failed with return code %d"
+ % (cmd, proc.returncode)
+ )
+ except OSError as exc:
+ raise self.severe(u"problems with '%s' directive: %s."
+ % (self.name, ErrorString(exc)))
+ return unicode(out)
+
+ def nestedParse(self, lines, fname):
+ content = ViewList()
+ node = nodes.section()
+
+ if "debug" in self.options:
+ code_block = "\n\n.. code-block:: rst\n :linenos:\n"
+ for l in lines.split("\n"):
+ code_block += "\n " + l
+ lines = code_block + "\n\n"
+
+ for c, l in enumerate(lines.split("\n")):
+ content.append(l, fname, c)
+
+ buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
+ self.state.memo.title_styles = []
+ self.state.memo.section_level = 0
+ self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter)
+ try:
+ self.state.nested_parse(content, 0, node, match_titles=1)
+ finally:
+ self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf
+ return node.children
--
2.21.0
^ permalink raw reply related
* [PATCH v3 17/20] docs: ABI: create a 2-depth index for ABI
From: Mauro Carvalho Chehab @ 2019-07-17 11:05 UTC (permalink / raw)
To: gregkh; +Cc: Mauro Carvalho Chehab, Jonathan Corbet, linux-doc
In-Reply-To: <cover.1563360659.git.mchehab+samsung@kernel.org>
That helps to identify what ABI files are adding titles.
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
---
Documentation/admin-guide/abi.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/admin-guide/abi.rst b/Documentation/admin-guide/abi.rst
index 3b9645c77469..bcab3ef2597c 100644
--- a/Documentation/admin-guide/abi.rst
+++ b/Documentation/admin-guide/abi.rst
@@ -3,7 +3,7 @@ Linux ABI description
=====================
.. toctree::
- :maxdepth: 1
+ :maxdepth: 2
abi-stable
abi-testing
--
2.21.0
^ permalink raw reply related
* [PATCH v3 07/20] docs: kernel_abi.py: make it compatible with Sphinx 1.7+
From: Mauro Carvalho Chehab @ 2019-07-17 11:05 UTC (permalink / raw)
To: gregkh; +Cc: Mauro Carvalho Chehab, Jonathan Corbet, linux-doc
In-Reply-To: <cover.1563360659.git.mchehab+samsung@kernel.org>
The same way kerneldoc.py needed changes to work with newer
Sphinx, this script needs the same changes.
While here, reorganize the include order to match kerneldoc.py.
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
---
Documentation/sphinx/kernel_abi.py | 39 +++++++++++++++++++++---------
1 file changed, 27 insertions(+), 12 deletions(-)
diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py
index 0f3e51e67e8d..2d5d582207f7 100644
--- a/Documentation/sphinx/kernel_abi.py
+++ b/Documentation/sphinx/kernel_abi.py
@@ -30,18 +30,27 @@ u"""
"""
import codecs
-import sys
import os
-from os import path
import subprocess
+import sys
-from sphinx.ext.autodoc import AutodocReporter
+from os import path
-from docutils import nodes
-from docutils.parsers.rst import Directive, directives
+from docutils import nodes, statemachine
from docutils.statemachine import ViewList
+from docutils.parsers.rst import directives, Directive
from docutils.utils.error_reporting import ErrorString
+#
+# AutodocReporter is only good up to Sphinx 1.7
+#
+import sphinx
+
+Use_SSI = sphinx.__version__[:3] >= '1.7'
+if Use_SSI:
+ from sphinx.util.docutils import switch_source_input
+else:
+ from sphinx.ext.autodoc import AutodocReporter
__version__ = '1.0'
@@ -139,11 +148,17 @@ class KernelCmd(Directive):
content.append(l, fname, c)
buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
- self.state.memo.title_styles = []
- self.state.memo.section_level = 0
- self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter)
- try:
- self.state.nested_parse(content, 0, node, match_titles=1)
- finally:
- self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf
+
+ if Use_SSI:
+ with switch_source_input(self.state, content):
+ self.state.nested_parse(content, 0, node, match_titles=1)
+ else:
+ self.state.memo.title_styles = []
+ self.state.memo.section_level = 0
+ self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter)
+ try:
+ self.state.nested_parse(content, 0, node, match_titles=1)
+ finally:
+ self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf
+
return node.children
--
2.21.0
^ permalink raw reply related
* [PATCH] docs: fs: convert porting to ReST
From: Mauro Carvalho Chehab @ 2019-07-17 10:46 UTC (permalink / raw)
Cc: Mauro Carvalho Chehab, Jonathan Corbet, Mike Marshall,
Martin Brandenburg, Al Viro, Dave Chinner, Ian Kent,
Darrick J. Wong, linux-doc, devel
Manually convert porting to ReST.
This file has its own proper style, except that, after a while,
its style gets violated and whitespaces are placed on
different creative ways.
As Sphinx and ReST are very sentitive to whitespace differences,
I had to opt if each entry after required/mandatory/... fields
should start with zero spaces or with a tab. I opted to start them
all from the zero position, in order to avoid needing to break lines
with more than 80 columns, with would make harder for review.
Most of the other changes at porting.rst were made to use a
notation with works nice as a text file while also produce a good html
output after being parsed.
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
---
This patch is against next-20190717
Documentation/filesystems/porting | 686 ---------------------
Documentation/filesystems/porting.rst | 841 ++++++++++++++++++++++++++
fs/orangefs/orangefs-kernel.h | 2 +-
3 files changed, 842 insertions(+), 687 deletions(-)
delete mode 100644 Documentation/filesystems/porting
create mode 100644 Documentation/filesystems/porting.rst
diff --git a/Documentation/filesystems/porting b/Documentation/filesystems/porting
deleted file mode 100644
index 209672010fb4..000000000000
--- a/Documentation/filesystems/porting
+++ /dev/null
@@ -1,686 +0,0 @@
-Changes since 2.5.0:
-
----
-[recommended]
-
-New helpers: sb_bread(), sb_getblk(), sb_find_get_block(), set_bh(),
- sb_set_blocksize() and sb_min_blocksize().
-
-Use them.
-
-(sb_find_get_block() replaces 2.4's get_hash_table())
-
----
-[recommended]
-
-New methods: ->alloc_inode() and ->destroy_inode().
-
-Remove inode->u.foo_inode_i
-Declare
- struct foo_inode_info {
- /* fs-private stuff */
- struct inode vfs_inode;
- };
- static inline struct foo_inode_info *FOO_I(struct inode *inode)
- {
- return list_entry(inode, struct foo_inode_info, vfs_inode);
- }
-
-Use FOO_I(inode) instead of &inode->u.foo_inode_i;
-
-Add foo_alloc_inode() and foo_destroy_inode() - the former should allocate
-foo_inode_info and return the address of ->vfs_inode, the latter should free
-FOO_I(inode) (see in-tree filesystems for examples).
-
-Make them ->alloc_inode and ->destroy_inode in your super_operations.
-
-Keep in mind that now you need explicit initialization of private data
-typically between calling iget_locked() and unlocking the inode.
-
-At some point that will become mandatory.
-
----
-[mandatory]
-
-Change of file_system_type method (->read_super to ->get_sb)
-
-->read_super() is no more. Ditto for DECLARE_FSTYPE and DECLARE_FSTYPE_DEV.
-
-Turn your foo_read_super() into a function that would return 0 in case of
-success and negative number in case of error (-EINVAL unless you have more
-informative error value to report). Call it foo_fill_super(). Now declare
-
-int foo_get_sb(struct file_system_type *fs_type,
- int flags, const char *dev_name, void *data, struct vfsmount *mnt)
-{
- return get_sb_bdev(fs_type, flags, dev_name, data, foo_fill_super,
- mnt);
-}
-
-(or similar with s/bdev/nodev/ or s/bdev/single/, depending on the kind of
-filesystem).
-
-Replace DECLARE_FSTYPE... with explicit initializer and have ->get_sb set as
-foo_get_sb.
-
----
-[mandatory]
-
-Locking change: ->s_vfs_rename_sem is taken only by cross-directory renames.
-Most likely there is no need to change anything, but if you relied on
-global exclusion between renames for some internal purpose - you need to
-change your internal locking. Otherwise exclusion warranties remain the
-same (i.e. parents and victim are locked, etc.).
-
----
-[informational]
-
-Now we have the exclusion between ->lookup() and directory removal (by
-->rmdir() and ->rename()). If you used to need that exclusion and do
-it by internal locking (most of filesystems couldn't care less) - you
-can relax your locking.
-
----
-[mandatory]
-
-->lookup(), ->truncate(), ->create(), ->unlink(), ->mknod(), ->mkdir(),
-->rmdir(), ->link(), ->lseek(), ->symlink(), ->rename()
-and ->readdir() are called without BKL now. Grab it on entry, drop upon return
-- that will guarantee the same locking you used to have. If your method or its
-parts do not need BKL - better yet, now you can shift lock_kernel() and
-unlock_kernel() so that they would protect exactly what needs to be
-protected.
-
----
-[mandatory]
-
-BKL is also moved from around sb operations. BKL should have been shifted into
-individual fs sb_op functions. If you don't need it, remove it.
-
----
-[informational]
-
-check for ->link() target not being a directory is done by callers. Feel
-free to drop it...
-
----
-[informational]
-
-->link() callers hold ->i_mutex on the object we are linking to. Some of your
-problems might be over...
-
----
-[mandatory]
-
-new file_system_type method - kill_sb(superblock). If you are converting
-an existing filesystem, set it according to ->fs_flags:
- FS_REQUIRES_DEV - kill_block_super
- FS_LITTER - kill_litter_super
- neither - kill_anon_super
-FS_LITTER is gone - just remove it from fs_flags.
-
----
-[mandatory]
-
- FS_SINGLE is gone (actually, that had happened back when ->get_sb()
-went in - and hadn't been documented ;-/). Just remove it from fs_flags
-(and see ->get_sb() entry for other actions).
-
----
-[mandatory]
-
-->setattr() is called without BKL now. Caller _always_ holds ->i_mutex, so
-watch for ->i_mutex-grabbing code that might be used by your ->setattr().
-Callers of notify_change() need ->i_mutex now.
-
----
-[recommended]
-
-New super_block field "struct export_operations *s_export_op" for
-explicit support for exporting, e.g. via NFS. The structure is fully
-documented at its declaration in include/linux/fs.h, and in
-Documentation/filesystems/nfs/Exporting.
-
-Briefly it allows for the definition of decode_fh and encode_fh operations
-to encode and decode filehandles, and allows the filesystem to use
-a standard helper function for decode_fh, and provide file-system specific
-support for this helper, particularly get_parent.
-
-It is planned that this will be required for exporting once the code
-settles down a bit.
-
-[mandatory]
-
-s_export_op is now required for exporting a filesystem.
-isofs, ext2, ext3, resierfs, fat
-can be used as examples of very different filesystems.
-
----
-[mandatory]
-
-iget4() and the read_inode2 callback have been superseded by iget5_locked()
-which has the following prototype,
-
- struct inode *iget5_locked(struct super_block *sb, unsigned long ino,
- int (*test)(struct inode *, void *),
- int (*set)(struct inode *, void *),
- void *data);
-
-'test' is an additional function that can be used when the inode
-number is not sufficient to identify the actual file object. 'set'
-should be a non-blocking function that initializes those parts of a
-newly created inode to allow the test function to succeed. 'data' is
-passed as an opaque value to both test and set functions.
-
-When the inode has been created by iget5_locked(), it will be returned with the
-I_NEW flag set and will still be locked. The filesystem then needs to finalize
-the initialization. Once the inode is initialized it must be unlocked by
-calling unlock_new_inode().
-
-The filesystem is responsible for setting (and possibly testing) i_ino
-when appropriate. There is also a simpler iget_locked function that
-just takes the superblock and inode number as arguments and does the
-test and set for you.
-
-e.g.
- inode = iget_locked(sb, ino);
- if (inode->i_state & I_NEW) {
- err = read_inode_from_disk(inode);
- if (err < 0) {
- iget_failed(inode);
- return err;
- }
- unlock_new_inode(inode);
- }
-
-Note that if the process of setting up a new inode fails, then iget_failed()
-should be called on the inode to render it dead, and an appropriate error
-should be passed back to the caller.
-
----
-[recommended]
-
-->getattr() finally getting used. See instances in nfs, minix, etc.
-
----
-[mandatory]
-
-->revalidate() is gone. If your filesystem had it - provide ->getattr()
-and let it call whatever you had as ->revlidate() + (for symlinks that
-had ->revalidate()) add calls in ->follow_link()/->readlink().
-
----
-[mandatory]
-
-->d_parent changes are not protected by BKL anymore. Read access is safe
-if at least one of the following is true:
- * filesystem has no cross-directory rename()
- * we know that parent had been locked (e.g. we are looking at
-->d_parent of ->lookup() argument).
- * we are called from ->rename().
- * the child's ->d_lock is held
-Audit your code and add locking if needed. Notice that any place that is
-not protected by the conditions above is risky even in the old tree - you
-had been relying on BKL and that's prone to screwups. Old tree had quite
-a few holes of that kind - unprotected access to ->d_parent leading to
-anything from oops to silent memory corruption.
-
----
-[mandatory]
-
- FS_NOMOUNT is gone. If you use it - just set SB_NOUSER in flags
-(see rootfs for one kind of solution and bdev/socket/pipe for another).
-
----
-[recommended]
-
- Use bdev_read_only(bdev) instead of is_read_only(kdev). The latter
-is still alive, but only because of the mess in drivers/s390/block/dasd.c.
-As soon as it gets fixed is_read_only() will die.
-
----
-[mandatory]
-
-->permission() is called without BKL now. Grab it on entry, drop upon
-return - that will guarantee the same locking you used to have. If
-your method or its parts do not need BKL - better yet, now you can
-shift lock_kernel() and unlock_kernel() so that they would protect
-exactly what needs to be protected.
-
----
-[mandatory]
-
-->statfs() is now called without BKL held. BKL should have been
-shifted into individual fs sb_op functions where it's not clear that
-it's safe to remove it. If you don't need it, remove it.
-
----
-[mandatory]
-
- is_read_only() is gone; use bdev_read_only() instead.
-
----
-[mandatory]
-
- destroy_buffers() is gone; use invalidate_bdev().
-
----
-[mandatory]
-
- fsync_dev() is gone; use fsync_bdev(). NOTE: lvm breakage is
-deliberate; as soon as struct block_device * is propagated in a reasonable
-way by that code fixing will become trivial; until then nothing can be
-done.
-
-[mandatory]
-
- block truncatation on error exit from ->write_begin, and ->direct_IO
-moved from generic methods (block_write_begin, cont_write_begin,
-nobh_write_begin, blockdev_direct_IO*) to callers. Take a look at
-ext2_write_failed and callers for an example.
-
-[mandatory]
-
- ->truncate is gone. The whole truncate sequence needs to be
-implemented in ->setattr, which is now mandatory for filesystems
-implementing on-disk size changes. Start with a copy of the old inode_setattr
-and vmtruncate, and the reorder the vmtruncate + foofs_vmtruncate sequence to
-be in order of zeroing blocks using block_truncate_page or similar helpers,
-size update and on finally on-disk truncation which should not fail.
-setattr_prepare (which used to be inode_change_ok) now includes the size checks
-for ATTR_SIZE and must be called in the beginning of ->setattr unconditionally.
-
-[mandatory]
-
- ->clear_inode() and ->delete_inode() are gone; ->evict_inode() should
-be used instead. It gets called whenever the inode is evicted, whether it has
-remaining links or not. Caller does *not* evict the pagecache or inode-associated
-metadata buffers; the method has to use truncate_inode_pages_final() to get rid
-of those. Caller makes sure async writeback cannot be running for the inode while
-(or after) ->evict_inode() is called.
-
- ->drop_inode() returns int now; it's called on final iput() with
-inode->i_lock held and it returns true if filesystems wants the inode to be
-dropped. As before, generic_drop_inode() is still the default and it's been
-updated appropriately. generic_delete_inode() is also alive and it consists
-simply of return 1. Note that all actual eviction work is done by caller after
-->drop_inode() returns.
-
- As before, clear_inode() must be called exactly once on each call of
-->evict_inode() (as it used to be for each call of ->delete_inode()). Unlike
-before, if you are using inode-associated metadata buffers (i.e.
-mark_buffer_dirty_inode()), it's your responsibility to call
-invalidate_inode_buffers() before clear_inode().
-
- NOTE: checking i_nlink in the beginning of ->write_inode() and bailing out
-if it's zero is not *and* *never* *had* *been* enough. Final unlink() and iput()
-may happen while the inode is in the middle of ->write_inode(); e.g. if you blindly
-free the on-disk inode, you may end up doing that while ->write_inode() is writing
-to it.
-
----
-[mandatory]
-
- .d_delete() now only advises the dcache as to whether or not to cache
-unreferenced dentries, and is now only called when the dentry refcount goes to
-0. Even on 0 refcount transition, it must be able to tolerate being called 0,
-1, or more times (eg. constant, idempotent).
-
----
-[mandatory]
-
- .d_compare() calling convention and locking rules are significantly
-changed. Read updated documentation in Documentation/filesystems/vfs.rst (and
-look at examples of other filesystems) for guidance.
-
----
-[mandatory]
-
- .d_hash() calling convention and locking rules are significantly
-changed. Read updated documentation in Documentation/filesystems/vfs.rst (and
-look at examples of other filesystems) for guidance.
-
----
-[mandatory]
- dcache_lock is gone, replaced by fine grained locks. See fs/dcache.c
-for details of what locks to replace dcache_lock with in order to protect
-particular things. Most of the time, a filesystem only needs ->d_lock, which
-protects *all* the dcache state of a given dentry.
-
---
-[mandatory]
-
- Filesystems must RCU-free their inodes, if they can have been accessed
-via rcu-walk path walk (basically, if the file can have had a path name in the
-vfs namespace).
-
- Even though i_dentry and i_rcu share storage in a union, we will
-initialize the former in inode_init_always(), so just leave it alone in
-the callback. It used to be necessary to clean it there, but not anymore
-(starting at 3.2).
-
---
-[recommended]
- vfs now tries to do path walking in "rcu-walk mode", which avoids
-atomic operations and scalability hazards on dentries and inodes (see
-Documentation/filesystems/path-lookup.txt). d_hash and d_compare changes
-(above) are examples of the changes required to support this. For more complex
-filesystem callbacks, the vfs drops out of rcu-walk mode before the fs call, so
-no changes are required to the filesystem. However, this is costly and loses
-the benefits of rcu-walk mode. We will begin to add filesystem callbacks that
-are rcu-walk aware, shown below. Filesystems should take advantage of this
-where possible.
-
---
-[mandatory]
- d_revalidate is a callback that is made on every path element (if
-the filesystem provides it), which requires dropping out of rcu-walk mode. This
-may now be called in rcu-walk mode (nd->flags & LOOKUP_RCU). -ECHILD should be
-returned if the filesystem cannot handle rcu-walk. See
-Documentation/filesystems/vfs.rst for more details.
-
- permission is an inode permission check that is called on many or all
-directory inodes on the way down a path walk (to check for exec permission). It
-must now be rcu-walk aware (mask & MAY_NOT_BLOCK). See
-Documentation/filesystems/vfs.rst for more details.
-
---
-[mandatory]
- In ->fallocate() you must check the mode option passed in. If your
-filesystem does not support hole punching (deallocating space in the middle of a
-file) you must return -EOPNOTSUPP if FALLOC_FL_PUNCH_HOLE is set in mode.
-Currently you can only have FALLOC_FL_PUNCH_HOLE with FALLOC_FL_KEEP_SIZE set,
-so the i_size should not change when hole punching, even when puching the end of
-a file off.
-
---
-[mandatory]
- ->get_sb() is gone. Switch to use of ->mount(). Typically it's just
-a matter of switching from calling get_sb_... to mount_... and changing the
-function type. If you were doing it manually, just switch from setting ->mnt_root
-to some pointer to returning that pointer. On errors return ERR_PTR(...).
-
---
-[mandatory]
- ->permission() and generic_permission()have lost flags
-argument; instead of passing IPERM_FLAG_RCU we add MAY_NOT_BLOCK into mask.
- generic_permission() has also lost the check_acl argument; ACL checking
-has been taken to VFS and filesystems need to provide a non-NULL ->i_op->get_acl
-to read an ACL from disk.
-
---
-[mandatory]
- If you implement your own ->llseek() you must handle SEEK_HOLE and
-SEEK_DATA. You can hanle this by returning -EINVAL, but it would be nicer to
-support it in some way. The generic handler assumes that the entire file is
-data and there is a virtual hole at the end of the file. So if the provided
-offset is less than i_size and SEEK_DATA is specified, return the same offset.
-If the above is true for the offset and you are given SEEK_HOLE, return the end
-of the file. If the offset is i_size or greater return -ENXIO in either case.
-
-[mandatory]
- If you have your own ->fsync() you must make sure to call
-filemap_write_and_wait_range() so that all dirty pages are synced out properly.
-You must also keep in mind that ->fsync() is not called with i_mutex held
-anymore, so if you require i_mutex locking you must make sure to take it and
-release it yourself.
-
---
-[mandatory]
- d_alloc_root() is gone, along with a lot of bugs caused by code
-misusing it. Replacement: d_make_root(inode). On success d_make_root(inode)
-allocates and returns a new dentry instantiated with the passed in inode.
-On failure NULL is returned and the passed in inode is dropped so the reference
-to inode is consumed in all cases and failure handling need not do any cleanup
-for the inode. If d_make_root(inode) is passed a NULL inode it returns NULL
-and also requires no further error handling. Typical usage is:
-
- inode = foofs_new_inode(....);
- s->s_root = d_make_inode(inode);
- if (!s->s_root)
- /* Nothing needed for the inode cleanup */
- return -ENOMEM;
- ...
-
---
-[mandatory]
- The witch is dead! Well, 2/3 of it, anyway. ->d_revalidate() and
-->lookup() do *not* take struct nameidata anymore; just the flags.
---
-[mandatory]
- ->create() doesn't take struct nameidata *; unlike the previous
-two, it gets "is it an O_EXCL or equivalent?" boolean argument. Note that
-local filesystems can ignore tha argument - they are guaranteed that the
-object doesn't exist. It's remote/distributed ones that might care...
---
-[mandatory]
- FS_REVAL_DOT is gone; if you used to have it, add ->d_weak_revalidate()
-in your dentry operations instead.
---
-[mandatory]
- vfs_readdir() is gone; switch to iterate_dir() instead
---
-[mandatory]
- ->readdir() is gone now; switch to ->iterate()
-[mandatory]
- vfs_follow_link has been removed. Filesystems must use nd_set_link
- from ->follow_link for normal symlinks, or nd_jump_link for magic
- /proc/<pid> style links.
---
-[mandatory]
- iget5_locked()/ilookup5()/ilookup5_nowait() test() callback used to be
- called with both ->i_lock and inode_hash_lock held; the former is *not*
- taken anymore, so verify that your callbacks do not rely on it (none
- of the in-tree instances did). inode_hash_lock is still held,
- of course, so they are still serialized wrt removal from inode hash,
- as well as wrt set() callback of iget5_locked().
---
-[mandatory]
- d_materialise_unique() is gone; d_splice_alias() does everything you
- need now. Remember that they have opposite orders of arguments ;-/
---
-[mandatory]
- f_dentry is gone; use f_path.dentry, or, better yet, see if you can avoid
- it entirely.
---
-[mandatory]
- never call ->read() and ->write() directly; use __vfs_{read,write} or
- wrappers; instead of checking for ->write or ->read being NULL, look for
- FMODE_CAN_{WRITE,READ} in file->f_mode.
---
-[mandatory]
- do _not_ use new_sync_{read,write} for ->read/->write; leave it NULL
- instead.
---
-[mandatory]
- ->aio_read/->aio_write are gone. Use ->read_iter/->write_iter.
----
-[recommended]
- for embedded ("fast") symlinks just set inode->i_link to wherever the
- symlink body is and use simple_follow_link() as ->follow_link().
---
-[mandatory]
- calling conventions for ->follow_link() have changed. Instead of returning
- cookie and using nd_set_link() to store the body to traverse, we return
- the body to traverse and store the cookie using explicit void ** argument.
- nameidata isn't passed at all - nd_jump_link() doesn't need it and
- nd_[gs]et_link() is gone.
---
-[mandatory]
- calling conventions for ->put_link() have changed. It gets inode instead of
- dentry, it does not get nameidata at all and it gets called only when cookie
- is non-NULL. Note that link body isn't available anymore, so if you need it,
- store it as cookie.
---
-[mandatory]
- any symlink that might use page_follow_link_light/page_put_link() must
- have inode_nohighmem(inode) called before anything might start playing with
- its pagecache. No highmem pages should end up in the pagecache of such
- symlinks. That includes any preseeding that might be done during symlink
- creation. __page_symlink() will honour the mapping gfp flags, so once
- you've done inode_nohighmem() it's safe to use, but if you allocate and
- insert the page manually, make sure to use the right gfp flags.
---
-[mandatory]
- ->follow_link() is replaced with ->get_link(); same API, except that
- * ->get_link() gets inode as a separate argument
- * ->get_link() may be called in RCU mode - in that case NULL
- dentry is passed
---
-[mandatory]
- ->get_link() gets struct delayed_call *done now, and should do
- set_delayed_call() where it used to set *cookie.
- ->put_link() is gone - just give the destructor to set_delayed_call()
- in ->get_link().
---
-[mandatory]
- ->getxattr() and xattr_handler.get() get dentry and inode passed separately.
- dentry might be yet to be attached to inode, so do _not_ use its ->d_inode
- in the instances. Rationale: !@#!@# security_d_instantiate() needs to be
- called before we attach dentry to inode.
---
-[mandatory]
- symlinks are no longer the only inodes that do *not* have i_bdev/i_cdev/
- i_pipe/i_link union zeroed out at inode eviction. As the result, you can't
- assume that non-NULL value in ->i_nlink at ->destroy_inode() implies that
- it's a symlink. Checking ->i_mode is really needed now. In-tree we had
- to fix shmem_destroy_callback() that used to take that kind of shortcut;
- watch out, since that shortcut is no longer valid.
---
-[mandatory]
- ->i_mutex is replaced with ->i_rwsem now. inode_lock() et.al. work as
- they used to - they just take it exclusive. However, ->lookup() may be
- called with parent locked shared. Its instances must not
- * use d_instantiate) and d_rehash() separately - use d_add() or
- d_splice_alias() instead.
- * use d_rehash() alone - call d_add(new_dentry, NULL) instead.
- * in the unlikely case when (read-only) access to filesystem
- data structures needs exclusion for some reason, arrange it
- yourself. None of the in-tree filesystems needed that.
- * rely on ->d_parent and ->d_name not changing after dentry has
- been fed to d_add() or d_splice_alias(). Again, none of the
- in-tree instances relied upon that.
- We are guaranteed that lookups of the same name in the same directory
- will not happen in parallel ("same" in the sense of your ->d_compare()).
- Lookups on different names in the same directory can and do happen in
- parallel now.
---
-[recommended]
- ->iterate_shared() is added; it's a parallel variant of ->iterate().
- Exclusion on struct file level is still provided (as well as that
- between it and lseek on the same struct file), but if your directory
- has been opened several times, you can get these called in parallel.
- Exclusion between that method and all directory-modifying ones is
- still provided, of course.
-
- Often enough ->iterate() can serve as ->iterate_shared() without any
- changes - it is a read-only operation, after all. If you have any
- per-inode or per-dentry in-core data structures modified by ->iterate(),
- you might need something to serialize the access to them. If you
- do dcache pre-seeding, you'll need to switch to d_alloc_parallel() for
- that; look for in-tree examples.
-
- Old method is only used if the new one is absent; eventually it will
- be removed. Switch while you still can; the old one won't stay.
---
-[mandatory]
- ->atomic_open() calls without O_CREAT may happen in parallel.
---
-[mandatory]
- ->setxattr() and xattr_handler.set() get dentry and inode passed separately.
- dentry might be yet to be attached to inode, so do _not_ use its ->d_inode
- in the instances. Rationale: !@#!@# security_d_instantiate() needs to be
- called before we attach dentry to inode and !@#!@##!@$!$#!@#$!@$!@$ smack
- ->d_instantiate() uses not just ->getxattr() but ->setxattr() as well.
---
-[mandatory]
- ->d_compare() doesn't get parent as a separate argument anymore. If you
- used it for finding the struct super_block involved, dentry->d_sb will
- work just as well; if it's something more complicated, use dentry->d_parent.
- Just be careful not to assume that fetching it more than once will yield
- the same value - in RCU mode it could change under you.
---
-[mandatory]
- ->rename() has an added flags argument. Any flags not handled by the
- filesystem should result in EINVAL being returned.
---
-[recommended]
- ->readlink is optional for symlinks. Don't set, unless filesystem needs
- to fake something for readlink(2).
---
-[mandatory]
- ->getattr() is now passed a struct path rather than a vfsmount and
- dentry separately, and it now has request_mask and query_flags arguments
- to specify the fields and sync type requested by statx. Filesystems not
- supporting any statx-specific features may ignore the new arguments.
---
-[mandatory]
- ->atomic_open() calling conventions have changed. Gone is int *opened,
- along with FILE_OPENED/FILE_CREATED. In place of those we have
- FMODE_OPENED/FMODE_CREATED, set in file->f_mode. Additionally, return
- value for 'called finish_no_open(), open it yourself' case has become
- 0, not 1. Since finish_no_open() itself is returning 0 now, that part
- does not need any changes in ->atomic_open() instances.
---
-[mandatory]
- alloc_file() has become static now; two wrappers are to be used instead.
- alloc_file_pseudo(inode, vfsmount, name, flags, ops) is for the cases
- when dentry needs to be created; that's the majority of old alloc_file()
- users. Calling conventions: on success a reference to new struct file
- is returned and callers reference to inode is subsumed by that. On
- failure, ERR_PTR() is returned and no caller's references are affected,
- so the caller needs to drop the inode reference it held.
- alloc_file_clone(file, flags, ops) does not affect any caller's references.
- On success you get a new struct file sharing the mount/dentry with the
- original, on failure - ERR_PTR().
---
-[mandatory]
- ->clone_file_range() and ->dedupe_file_range have been replaced with
- ->remap_file_range(). See Documentation/filesystems/vfs.rst for more
- information.
---
-[recommended]
- ->lookup() instances doing an equivalent of
- if (IS_ERR(inode))
- return ERR_CAST(inode);
- return d_splice_alias(inode, dentry);
- don't need to bother with the check - d_splice_alias() will do the
- right thing when given ERR_PTR(...) as inode. Moreover, passing NULL
- inode to d_splice_alias() will also do the right thing (equivalent of
- d_add(dentry, NULL); return NULL;), so that kind of special cases
- also doesn't need a separate treatment.
---
-[strongly recommended]
- take the RCU-delayed parts of ->destroy_inode() into a new method -
- ->free_inode(). If ->destroy_inode() becomes empty - all the better,
- just get rid of it. Synchronous work (e.g. the stuff that can't
- be done from an RCU callback, or any WARN_ON() where we want the
- stack trace) *might* be movable to ->evict_inode(); however,
- that goes only for the things that are not needed to balance something
- done by ->alloc_inode(). IOW, if it's cleaning up the stuff that
- might have accumulated over the life of in-core inode, ->evict_inode()
- might be a fit.
-
- Rules for inode destruction:
- * if ->destroy_inode() is non-NULL, it gets called
- * if ->free_inode() is non-NULL, it gets scheduled by call_rcu()
- * combination of NULL ->destroy_inode and NULL ->free_inode is
- treated as NULL/free_inode_nonrcu, to preserve the compatibility.
-
- Note that the callback (be it via ->free_inode() or explicit call_rcu()
- in ->destroy_inode()) is *NOT* ordered wrt superblock destruction;
- as the matter of fact, the superblock and all associated structures
- might be already gone. The filesystem driver is guaranteed to be still
- there, but that's it. Freeing memory in the callback is fine; doing
- more than that is possible, but requires a lot of care and is best
- avoided.
---
-[mandatory]
- DCACHE_RCUACCESS is gone; having an RCU delay on dentry freeing is the
- default. DCACHE_NORCU opts out, and only d_alloc_pseudo() has any
- business doing so.
---
-[mandatory]
- d_alloc_pseudo() is internal-only; uses outside of alloc_file_pseudo() are
- very suspect (and won't work in modules). Such uses are very likely to
- be misspelled d_alloc_anon().
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
new file mode 100644
index 000000000000..73913d7b9c5f
--- /dev/null
+++ b/Documentation/filesystems/porting.rst
@@ -0,0 +1,841 @@
+====================
+Changes since 2.5.0:
+====================
+
+---
+
+**recommended**
+
+New helpers: sb_bread(), sb_getblk(), sb_find_get_block(), set_bh(),
+sb_set_blocksize() and sb_min_blocksize().
+
+Use them.
+
+(sb_find_get_block() replaces 2.4's get_hash_table())
+
+---
+
+**recommended**
+
+New methods: ->alloc_inode() and ->destroy_inode().
+
+Remove inode->u.foo_inode_i
+
+Declare::
+
+ struct foo_inode_info {
+ /* fs-private stuff */
+ struct inode vfs_inode;
+ };
+ static inline struct foo_inode_info *FOO_I(struct inode *inode)
+ {
+ return list_entry(inode, struct foo_inode_info, vfs_inode);
+ }
+
+Use FOO_I(inode) instead of &inode->u.foo_inode_i;
+
+Add foo_alloc_inode() and foo_destroy_inode() - the former should allocate
+foo_inode_info and return the address of ->vfs_inode, the latter should free
+FOO_I(inode) (see in-tree filesystems for examples).
+
+Make them ->alloc_inode and ->destroy_inode in your super_operations.
+
+Keep in mind that now you need explicit initialization of private data
+typically between calling iget_locked() and unlocking the inode.
+
+At some point that will become mandatory.
+
+---
+
+**mandatory**
+
+Change of file_system_type method (->read_super to ->get_sb)
+
+->read_super() is no more. Ditto for DECLARE_FSTYPE and DECLARE_FSTYPE_DEV.
+
+Turn your foo_read_super() into a function that would return 0 in case of
+success and negative number in case of error (-EINVAL unless you have more
+informative error value to report). Call it foo_fill_super(). Now declare::
+
+ int foo_get_sb(struct file_system_type *fs_type,
+ int flags, const char *dev_name, void *data, struct vfsmount *mnt)
+ {
+ return get_sb_bdev(fs_type, flags, dev_name, data, foo_fill_super,
+ mnt);
+ }
+
+(or similar with s/bdev/nodev/ or s/bdev/single/, depending on the kind of
+filesystem).
+
+Replace DECLARE_FSTYPE... with explicit initializer and have ->get_sb set as
+foo_get_sb.
+
+---
+
+**mandatory**
+
+Locking change: ->s_vfs_rename_sem is taken only by cross-directory renames.
+Most likely there is no need to change anything, but if you relied on
+global exclusion between renames for some internal purpose - you need to
+change your internal locking. Otherwise exclusion warranties remain the
+same (i.e. parents and victim are locked, etc.).
+
+---
+
+**informational**
+
+Now we have the exclusion between ->lookup() and directory removal (by
+->rmdir() and ->rename()). If you used to need that exclusion and do
+it by internal locking (most of filesystems couldn't care less) - you
+can relax your locking.
+
+---
+
+**mandatory**
+
+->lookup(), ->truncate(), ->create(), ->unlink(), ->mknod(), ->mkdir(),
+->rmdir(), ->link(), ->lseek(), ->symlink(), ->rename()
+and ->readdir() are called without BKL now. Grab it on entry, drop upon return
+- that will guarantee the same locking you used to have. If your method or its
+parts do not need BKL - better yet, now you can shift lock_kernel() and
+unlock_kernel() so that they would protect exactly what needs to be
+protected.
+
+---
+
+**mandatory**
+
+BKL is also moved from around sb operations. BKL should have been shifted into
+individual fs sb_op functions. If you don't need it, remove it.
+
+---
+
+**informational**
+
+check for ->link() target not being a directory is done by callers. Feel
+free to drop it...
+
+---
+
+**informational**
+
+->link() callers hold ->i_mutex on the object we are linking to. Some of your
+problems might be over...
+
+---
+
+**mandatory**
+
+new file_system_type method - kill_sb(superblock). If you are converting
+an existing filesystem, set it according to ->fs_flags::
+
+ FS_REQUIRES_DEV - kill_block_super
+ FS_LITTER - kill_litter_super
+ neither - kill_anon_super
+
+FS_LITTER is gone - just remove it from fs_flags.
+
+---
+
+**mandatory**
+
+FS_SINGLE is gone (actually, that had happened back when ->get_sb()
+went in - and hadn't been documented ;-/). Just remove it from fs_flags
+(and see ->get_sb() entry for other actions).
+
+---
+
+**mandatory**
+
+->setattr() is called without BKL now. Caller _always_ holds ->i_mutex, so
+watch for ->i_mutex-grabbing code that might be used by your ->setattr().
+Callers of notify_change() need ->i_mutex now.
+
+---
+
+**recommended**
+
+New super_block field ``struct export_operations *s_export_op`` for
+explicit support for exporting, e.g. via NFS. The structure is fully
+documented at its declaration in include/linux/fs.h, and in
+Documentation/filesystems/nfs/Exporting.
+
+Briefly it allows for the definition of decode_fh and encode_fh operations
+to encode and decode filehandles, and allows the filesystem to use
+a standard helper function for decode_fh, and provide file-system specific
+support for this helper, particularly get_parent.
+
+It is planned that this will be required for exporting once the code
+settles down a bit.
+
+**mandatory**
+
+s_export_op is now required for exporting a filesystem.
+isofs, ext2, ext3, resierfs, fat
+can be used as examples of very different filesystems.
+
+---
+
+**mandatory**
+
+iget4() and the read_inode2 callback have been superseded by iget5_locked()
+which has the following prototype::
+
+ struct inode *iget5_locked(struct super_block *sb, unsigned long ino,
+ int (*test)(struct inode *, void *),
+ int (*set)(struct inode *, void *),
+ void *data);
+
+'test' is an additional function that can be used when the inode
+number is not sufficient to identify the actual file object. 'set'
+should be a non-blocking function that initializes those parts of a
+newly created inode to allow the test function to succeed. 'data' is
+passed as an opaque value to both test and set functions.
+
+When the inode has been created by iget5_locked(), it will be returned with the
+I_NEW flag set and will still be locked. The filesystem then needs to finalize
+the initialization. Once the inode is initialized it must be unlocked by
+calling unlock_new_inode().
+
+The filesystem is responsible for setting (and possibly testing) i_ino
+when appropriate. There is also a simpler iget_locked function that
+just takes the superblock and inode number as arguments and does the
+test and set for you.
+
+e.g.::
+
+ inode = iget_locked(sb, ino);
+ if (inode->i_state & I_NEW) {
+ err = read_inode_from_disk(inode);
+ if (err < 0) {
+ iget_failed(inode);
+ return err;
+ }
+ unlock_new_inode(inode);
+ }
+
+Note that if the process of setting up a new inode fails, then iget_failed()
+should be called on the inode to render it dead, and an appropriate error
+should be passed back to the caller.
+
+---
+
+**recommended**
+
+->getattr() finally getting used. See instances in nfs, minix, etc.
+
+---
+
+**mandatory**
+
+->revalidate() is gone. If your filesystem had it - provide ->getattr()
+and let it call whatever you had as ->revlidate() + (for symlinks that
+had ->revalidate()) add calls in ->follow_link()/->readlink().
+
+---
+
+**mandatory**
+
+->d_parent changes are not protected by BKL anymore. Read access is safe
+if at least one of the following is true:
+
+ * filesystem has no cross-directory rename()
+ * we know that parent had been locked (e.g. we are looking at
+ ->d_parent of ->lookup() argument).
+ * we are called from ->rename().
+ * the child's ->d_lock is held
+
+Audit your code and add locking if needed. Notice that any place that is
+not protected by the conditions above is risky even in the old tree - you
+had been relying on BKL and that's prone to screwups. Old tree had quite
+a few holes of that kind - unprotected access to ->d_parent leading to
+anything from oops to silent memory corruption.
+
+---
+
+**mandatory**
+
+FS_NOMOUNT is gone. If you use it - just set SB_NOUSER in flags
+(see rootfs for one kind of solution and bdev/socket/pipe for another).
+
+---
+
+**recommended**
+
+Use bdev_read_only(bdev) instead of is_read_only(kdev). The latter
+is still alive, but only because of the mess in drivers/s390/block/dasd.c.
+As soon as it gets fixed is_read_only() will die.
+
+---
+
+**mandatory**
+
+->permission() is called without BKL now. Grab it on entry, drop upon
+return - that will guarantee the same locking you used to have. If
+your method or its parts do not need BKL - better yet, now you can
+shift lock_kernel() and unlock_kernel() so that they would protect
+exactly what needs to be protected.
+
+---
+
+**mandatory**
+
+->statfs() is now called without BKL held. BKL should have been
+shifted into individual fs sb_op functions where it's not clear that
+it's safe to remove it. If you don't need it, remove it.
+
+---
+
+**mandatory**
+
+is_read_only() is gone; use bdev_read_only() instead.
+
+---
+
+**mandatory**
+
+destroy_buffers() is gone; use invalidate_bdev().
+
+---
+
+**mandatory**
+
+fsync_dev() is gone; use fsync_bdev(). NOTE: lvm breakage is
+deliberate; as soon as struct block_device * is propagated in a reasonable
+way by that code fixing will become trivial; until then nothing can be
+done.
+
+**mandatory**
+
+block truncatation on error exit from ->write_begin, and ->direct_IO
+moved from generic methods (block_write_begin, cont_write_begin,
+nobh_write_begin, blockdev_direct_IO*) to callers. Take a look at
+ext2_write_failed and callers for an example.
+
+**mandatory**
+
+->truncate is gone. The whole truncate sequence needs to be
+implemented in ->setattr, which is now mandatory for filesystems
+implementing on-disk size changes. Start with a copy of the old inode_setattr
+and vmtruncate, and the reorder the vmtruncate + foofs_vmtruncate sequence to
+be in order of zeroing blocks using block_truncate_page or similar helpers,
+size update and on finally on-disk truncation which should not fail.
+setattr_prepare (which used to be inode_change_ok) now includes the size checks
+for ATTR_SIZE and must be called in the beginning of ->setattr unconditionally.
+
+**mandatory**
+
+->clear_inode() and ->delete_inode() are gone; ->evict_inode() should
+be used instead. It gets called whenever the inode is evicted, whether it has
+remaining links or not. Caller does *not* evict the pagecache or inode-associated
+metadata buffers; the method has to use truncate_inode_pages_final() to get rid
+of those. Caller makes sure async writeback cannot be running for the inode while
+(or after) ->evict_inode() is called.
+
+->drop_inode() returns int now; it's called on final iput() with
+inode->i_lock held and it returns true if filesystems wants the inode to be
+dropped. As before, generic_drop_inode() is still the default and it's been
+updated appropriately. generic_delete_inode() is also alive and it consists
+simply of return 1. Note that all actual eviction work is done by caller after
+->drop_inode() returns.
+
+As before, clear_inode() must be called exactly once on each call of
+->evict_inode() (as it used to be for each call of ->delete_inode()). Unlike
+before, if you are using inode-associated metadata buffers (i.e.
+mark_buffer_dirty_inode()), it's your responsibility to call
+invalidate_inode_buffers() before clear_inode().
+
+NOTE: checking i_nlink in the beginning of ->write_inode() and bailing out
+if it's zero is not *and* *never* *had* *been* enough. Final unlink() and iput()
+may happen while the inode is in the middle of ->write_inode(); e.g. if you blindly
+free the on-disk inode, you may end up doing that while ->write_inode() is writing
+to it.
+
+---
+
+**mandatory**
+
+.d_delete() now only advises the dcache as to whether or not to cache
+unreferenced dentries, and is now only called when the dentry refcount goes to
+0. Even on 0 refcount transition, it must be able to tolerate being called 0,
+1, or more times (eg. constant, idempotent).
+
+---
+
+**mandatory**
+
+.d_compare() calling convention and locking rules are significantly
+changed. Read updated documentation in Documentation/filesystems/vfs.rst (and
+look at examples of other filesystems) for guidance.
+
+---
+
+**mandatory**
+
+.d_hash() calling convention and locking rules are significantly
+changed. Read updated documentation in Documentation/filesystems/vfs.rst (and
+look at examples of other filesystems) for guidance.
+
+---
+
+**mandatory**
+
+dcache_lock is gone, replaced by fine grained locks. See fs/dcache.c
+for details of what locks to replace dcache_lock with in order to protect
+particular things. Most of the time, a filesystem only needs ->d_lock, which
+protects *all* the dcache state of a given dentry.
+
+---
+
+**mandatory**
+
+Filesystems must RCU-free their inodes, if they can have been accessed
+via rcu-walk path walk (basically, if the file can have had a path name in the
+vfs namespace).
+
+Even though i_dentry and i_rcu share storage in a union, we will
+initialize the former in inode_init_always(), so just leave it alone in
+the callback. It used to be necessary to clean it there, but not anymore
+(starting at 3.2).
+
+---
+
+**recommended**
+
+vfs now tries to do path walking in "rcu-walk mode", which avoids
+atomic operations and scalability hazards on dentries and inodes (see
+Documentation/filesystems/path-lookup.txt). d_hash and d_compare changes
+(above) are examples of the changes required to support this. For more complex
+filesystem callbacks, the vfs drops out of rcu-walk mode before the fs call, so
+no changes are required to the filesystem. However, this is costly and loses
+the benefits of rcu-walk mode. We will begin to add filesystem callbacks that
+are rcu-walk aware, shown below. Filesystems should take advantage of this
+where possible.
+
+---
+
+**mandatory**
+
+d_revalidate is a callback that is made on every path element (if
+the filesystem provides it), which requires dropping out of rcu-walk mode. This
+may now be called in rcu-walk mode (nd->flags & LOOKUP_RCU). -ECHILD should be
+returned if the filesystem cannot handle rcu-walk. See
+Documentation/filesystems/vfs.rst for more details.
+
+permission is an inode permission check that is called on many or all
+directory inodes on the way down a path walk (to check for exec permission). It
+must now be rcu-walk aware (mask & MAY_NOT_BLOCK). See
+Documentation/filesystems/vfs.rst for more details.
+
+---
+
+**mandatory**
+
+In ->fallocate() you must check the mode option passed in. If your
+filesystem does not support hole punching (deallocating space in the middle of a
+file) you must return -EOPNOTSUPP if FALLOC_FL_PUNCH_HOLE is set in mode.
+Currently you can only have FALLOC_FL_PUNCH_HOLE with FALLOC_FL_KEEP_SIZE set,
+so the i_size should not change when hole punching, even when puching the end of
+a file off.
+
+---
+
+**mandatory**
+
+->get_sb() is gone. Switch to use of ->mount(). Typically it's just
+a matter of switching from calling ``get_sb_``... to ``mount_``... and changing
+the function type. If you were doing it manually, just switch from setting
+->mnt_root to some pointer to returning that pointer. On errors return
+ERR_PTR(...).
+
+---
+
+**mandatory**
+
+->permission() and generic_permission()have lost flags
+argument; instead of passing IPERM_FLAG_RCU we add MAY_NOT_BLOCK into mask.
+
+generic_permission() has also lost the check_acl argument; ACL checking
+has been taken to VFS and filesystems need to provide a non-NULL ->i_op->get_acl
+to read an ACL from disk.
+
+---
+
+**mandatory**
+
+If you implement your own ->llseek() you must handle SEEK_HOLE and
+SEEK_DATA. You can hanle this by returning -EINVAL, but it would be nicer to
+support it in some way. The generic handler assumes that the entire file is
+data and there is a virtual hole at the end of the file. So if the provided
+offset is less than i_size and SEEK_DATA is specified, return the same offset.
+If the above is true for the offset and you are given SEEK_HOLE, return the end
+of the file. If the offset is i_size or greater return -ENXIO in either case.
+
+**mandatory**
+
+If you have your own ->fsync() you must make sure to call
+filemap_write_and_wait_range() so that all dirty pages are synced out properly.
+You must also keep in mind that ->fsync() is not called with i_mutex held
+anymore, so if you require i_mutex locking you must make sure to take it and
+release it yourself.
+
+---
+
+**mandatory**
+
+d_alloc_root() is gone, along with a lot of bugs caused by code
+misusing it. Replacement: d_make_root(inode). The difference is,
+d_make_root() drops the reference to inode if dentry allocation fails.
+
+---
+
+**mandatory**
+
+The witch is dead! Well, 2/3 of it, anyway. ->d_revalidate() and
+->lookup() do *not* take struct nameidata anymore; just the flags.
+
+---
+
+**mandatory**
+
+->create() doesn't take ``struct nameidata *``; unlike the previous
+two, it gets "is it an O_EXCL or equivalent?" boolean argument. Note that
+local filesystems can ignore tha argument - they are guaranteed that the
+object doesn't exist. It's remote/distributed ones that might care...
+
+---
+
+**mandatory**
+
+FS_REVAL_DOT is gone; if you used to have it, add ->d_weak_revalidate()
+in your dentry operations instead.
+
+---
+
+**mandatory**
+
+vfs_readdir() is gone; switch to iterate_dir() instead
+
+---
+
+**mandatory**
+
+->readdir() is gone now; switch to ->iterate()
+
+**mandatory**
+
+vfs_follow_link has been removed. Filesystems must use nd_set_link
+from ->follow_link for normal symlinks, or nd_jump_link for magic
+/proc/<pid> style links.
+
+---
+
+**mandatory**
+
+iget5_locked()/ilookup5()/ilookup5_nowait() test() callback used to be
+called with both ->i_lock and inode_hash_lock held; the former is *not*
+taken anymore, so verify that your callbacks do not rely on it (none
+of the in-tree instances did). inode_hash_lock is still held,
+of course, so they are still serialized wrt removal from inode hash,
+as well as wrt set() callback of iget5_locked().
+
+---
+
+**mandatory**
+
+d_materialise_unique() is gone; d_splice_alias() does everything you
+need now. Remember that they have opposite orders of arguments ;-/
+
+---
+
+**mandatory**
+
+f_dentry is gone; use f_path.dentry, or, better yet, see if you can avoid
+it entirely.
+
+---
+
+**mandatory**
+
+never call ->read() and ->write() directly; use __vfs_{read,write} or
+wrappers; instead of checking for ->write or ->read being NULL, look for
+FMODE_CAN_{WRITE,READ} in file->f_mode.
+
+---
+
+**mandatory**
+
+do _not_ use new_sync_{read,write} for ->read/->write; leave it NULL
+instead.
+
+---
+
+**mandatory**
+ ->aio_read/->aio_write are gone. Use ->read_iter/->write_iter.
+
+---
+
+**recommended**
+
+for embedded ("fast") symlinks just set inode->i_link to wherever the
+symlink body is and use simple_follow_link() as ->follow_link().
+
+---
+
+**mandatory**
+
+calling conventions for ->follow_link() have changed. Instead of returning
+cookie and using nd_set_link() to store the body to traverse, we return
+the body to traverse and store the cookie using explicit void ** argument.
+nameidata isn't passed at all - nd_jump_link() doesn't need it and
+nd_[gs]et_link() is gone.
+
+---
+
+**mandatory**
+
+calling conventions for ->put_link() have changed. It gets inode instead of
+dentry, it does not get nameidata at all and it gets called only when cookie
+is non-NULL. Note that link body isn't available anymore, so if you need it,
+store it as cookie.
+
+---
+
+**mandatory**
+
+any symlink that might use page_follow_link_light/page_put_link() must
+have inode_nohighmem(inode) called before anything might start playing with
+its pagecache. No highmem pages should end up in the pagecache of such
+symlinks. That includes any preseeding that might be done during symlink
+creation. __page_symlink() will honour the mapping gfp flags, so once
+you've done inode_nohighmem() it's safe to use, but if you allocate and
+insert the page manually, make sure to use the right gfp flags.
+
+---
+
+**mandatory**
+
+->follow_link() is replaced with ->get_link(); same API, except that
+
+ * ->get_link() gets inode as a separate argument
+ * ->get_link() may be called in RCU mode - in that case NULL
+ dentry is passed
+
+---
+
+**mandatory**
+
+->get_link() gets struct delayed_call ``*done`` now, and should do
+set_delayed_call() where it used to set ``*cookie``.
+
+->put_link() is gone - just give the destructor to set_delayed_call()
+in ->get_link().
+
+---
+
+**mandatory**
+
+->getxattr() and xattr_handler.get() get dentry and inode passed separately.
+dentry might be yet to be attached to inode, so do _not_ use its ->d_inode
+in the instances. Rationale: !@#!@# security_d_instantiate() needs to be
+called before we attach dentry to inode.
+
+---
+
+**mandatory**
+
+symlinks are no longer the only inodes that do *not* have i_bdev/i_cdev/
+i_pipe/i_link union zeroed out at inode eviction. As the result, you can't
+assume that non-NULL value in ->i_nlink at ->destroy_inode() implies that
+it's a symlink. Checking ->i_mode is really needed now. In-tree we had
+to fix shmem_destroy_callback() that used to take that kind of shortcut;
+watch out, since that shortcut is no longer valid.
+
+---
+
+**mandatory**
+
+->i_mutex is replaced with ->i_rwsem now. inode_lock() et.al. work as
+they used to - they just take it exclusive. However, ->lookup() may be
+called with parent locked shared. Its instances must not
+
+ * use d_instantiate) and d_rehash() separately - use d_add() or
+ d_splice_alias() instead.
+ * use d_rehash() alone - call d_add(new_dentry, NULL) instead.
+ * in the unlikely case when (read-only) access to filesystem
+ data structures needs exclusion for some reason, arrange it
+ yourself. None of the in-tree filesystems needed that.
+ * rely on ->d_parent and ->d_name not changing after dentry has
+ been fed to d_add() or d_splice_alias(). Again, none of the
+ in-tree instances relied upon that.
+
+We are guaranteed that lookups of the same name in the same directory
+will not happen in parallel ("same" in the sense of your ->d_compare()).
+Lookups on different names in the same directory can and do happen in
+parallel now.
+
+---
+
+**recommended**
+
+->iterate_shared() is added; it's a parallel variant of ->iterate().
+Exclusion on struct file level is still provided (as well as that
+between it and lseek on the same struct file), but if your directory
+has been opened several times, you can get these called in parallel.
+Exclusion between that method and all directory-modifying ones is
+still provided, of course.
+
+Often enough ->iterate() can serve as ->iterate_shared() without any
+changes - it is a read-only operation, after all. If you have any
+per-inode or per-dentry in-core data structures modified by ->iterate(),
+you might need something to serialize the access to them. If you
+do dcache pre-seeding, you'll need to switch to d_alloc_parallel() for
+that; look for in-tree examples.
+
+Old method is only used if the new one is absent; eventually it will
+be removed. Switch while you still can; the old one won't stay.
+
+---
+
+**mandatory**
+
+->atomic_open() calls without O_CREAT may happen in parallel.
+
+---
+
+**mandatory**
+
+->setxattr() and xattr_handler.set() get dentry and inode passed separately.
+dentry might be yet to be attached to inode, so do _not_ use its ->d_inode
+in the instances. Rationale: !@#!@# security_d_instantiate() needs to be
+called before we attach dentry to inode and !@#!@##!@$!$#!@#$!@$!@$ smack
+->d_instantiate() uses not just ->getxattr() but ->setxattr() as well.
+
+---
+
+**mandatory**
+
+->d_compare() doesn't get parent as a separate argument anymore. If you
+used it for finding the struct super_block involved, dentry->d_sb will
+work just as well; if it's something more complicated, use dentry->d_parent.
+Just be careful not to assume that fetching it more than once will yield
+the same value - in RCU mode it could change under you.
+
+---
+
+**mandatory**
+
+->rename() has an added flags argument. Any flags not handled by the
+filesystem should result in EINVAL being returned.
+
+---
+
+
+**recommended**
+
+->readlink is optional for symlinks. Don't set, unless filesystem needs
+to fake something for readlink(2).
+
+---
+
+**mandatory**
+
+->getattr() is now passed a struct path rather than a vfsmount and
+dentry separately, and it now has request_mask and query_flags arguments
+to specify the fields and sync type requested by statx. Filesystems not
+supporting any statx-specific features may ignore the new arguments.
+
+---
+
+**mandatory**
+
+->atomic_open() calling conventions have changed. Gone is ``int *opened``,
+along with FILE_OPENED/FILE_CREATED. In place of those we have
+FMODE_OPENED/FMODE_CREATED, set in file->f_mode. Additionally, return
+value for 'called finish_no_open(), open it yourself' case has become
+0, not 1. Since finish_no_open() itself is returning 0 now, that part
+does not need any changes in ->atomic_open() instances.
+
+---
+
+**mandatory**
+
+alloc_file() has become static now; two wrappers are to be used instead.
+alloc_file_pseudo(inode, vfsmount, name, flags, ops) is for the cases
+when dentry needs to be created; that's the majority of old alloc_file()
+users. Calling conventions: on success a reference to new struct file
+is returned and callers reference to inode is subsumed by that. On
+failure, ERR_PTR() is returned and no caller's references are affected,
+so the caller needs to drop the inode reference it held.
+alloc_file_clone(file, flags, ops) does not affect any caller's references.
+On success you get a new struct file sharing the mount/dentry with the
+original, on failure - ERR_PTR().
+
+---
+
+**mandatory**
+
+->clone_file_range() and ->dedupe_file_range have been replaced with
+->remap_file_range(). See Documentation/filesystems/vfs.rst for more
+information.
+
+---
+
+**recommended**
+
+->lookup() instances doing an equivalent of::
+
+ if (IS_ERR(inode))
+ return ERR_CAST(inode);
+ return d_splice_alias(inode, dentry);
+
+don't need to bother with the check - d_splice_alias() will do the
+right thing when given ERR_PTR(...) as inode. Moreover, passing NULL
+inode to d_splice_alias() will also do the right thing (equivalent of
+d_add(dentry, NULL); return NULL;), so that kind of special cases
+also doesn't need a separate treatment.
+
+---
+
+**strongly recommended**
+
+take the RCU-delayed parts of ->destroy_inode() into a new method -
+->free_inode(). If ->destroy_inode() becomes empty - all the better,
+just get rid of it. Synchronous work (e.g. the stuff that can't
+be done from an RCU callback, or any WARN_ON() where we want the
+stack trace) *might* be movable to ->evict_inode(); however,
+that goes only for the things that are not needed to balance something
+done by ->alloc_inode(). IOW, if it's cleaning up the stuff that
+might have accumulated over the life of in-core inode, ->evict_inode()
+might be a fit.
+
+Rules for inode destruction:
+
+ * if ->destroy_inode() is non-NULL, it gets called
+ * if ->free_inode() is non-NULL, it gets scheduled by call_rcu()
+ * combination of NULL ->destroy_inode and NULL ->free_inode is
+ treated as NULL/free_inode_nonrcu, to preserve the compatibility.
+
+Note that the callback (be it via ->free_inode() or explicit call_rcu()
+in ->destroy_inode()) is *NOT* ordered wrt superblock destruction;
+as the matter of fact, the superblock and all associated structures
+might be already gone. The filesystem driver is guaranteed to be still
+there, but that's it. Freeing memory in the callback is fine; doing
+more than that is possible, but requires a lot of care and is best
+avoided.
+
+---
+
+**mandatory**
+
+DCACHE_RCUACCESS is gone; having an RCU delay on dentry freeing is the
+default. DCACHE_NORCU opts out, and only d_alloc_pseudo() has any
+business doing so.
+
+---
+
+**mandatory**
+
+d_alloc_pseudo() is internal-only; uses outside of alloc_file_pseudo() are
+very suspect (and won't work in modules). Such uses are very likely to
+be misspelled d_alloc_anon().
diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h
index 58158d286a0c..5a42b35c3e17 100644
--- a/fs/orangefs/orangefs-kernel.h
+++ b/fs/orangefs/orangefs-kernel.h
@@ -248,7 +248,7 @@ struct orangefs_read_options {
extern struct orangefs_stats orangefs_stats;
/*
- * NOTE: See Documentation/filesystems/porting for information
+ * NOTE: See Documentation/filesystems/porting.rst for information
* on implementing FOO_I and properly accessing fs private data
*/
static inline struct orangefs_inode_s *ORANGEFS_I(struct inode *inode)
--
2.21.0
^ permalink raw reply related
* Re: [PATCH v2 2/2] f2fs: Support case-insensitive file name lookups
From: Chao Yu @ 2019-07-17 10:11 UTC (permalink / raw)
To: Daniel Rosenberg, Jaegeuk Kim, Jonathan Corbet, linux-f2fs-devel
Cc: linux-kernel, linux-doc, linux-fsdevel, kernel-team
In-Reply-To: <20190717031408.114104-3-drosen@google.com>
Hi Daniel,
On 2019/7/17 11:14, Daniel Rosenberg wrote:
> Modeled after commit b886ee3e778e ("ext4: Support case-insensitive file
> name lookups")
>
> """
> This patch implements the actual support for case-insensitive file name
> lookups in f2fs, based on the feature bit and the encoding stored in the
> superblock.
>
> A filesystem that has the casefold feature set is able to configure
> directories with the +F (F2FS_CASEFOLD_FL) attribute, enabling lookups
> to succeed in that directory in a case-insensitive fashion, i.e: match
> a directory entry even if the name used by userspace is not a byte per
> byte match with the disk name, but is an equivalent case-insensitive
> version of the Unicode string. This operation is called a
> case-insensitive file name lookup.
>
> The feature is configured as an inode attribute applied to directories
> and inherited by its children. This attribute can only be enabled on
> empty directories for filesystems that support the encoding feature,
> thus preventing collision of file names that only differ by case.
>
> * dcache handling:
>
> For a +F directory, F2Fs only stores the first equivalent name dentry
> used in the dcache. This is done to prevent unintentional duplication of
> dentries in the dcache, while also allowing the VFS code to quickly find
> the right entry in the cache despite which equivalent string was used in
> a previous lookup, without having to resort to ->lookup().
>
> d_hash() of casefolded directories is implemented as the hash of the
> casefolded string, such that we always have a well-known bucket for all
> the equivalencies of the same string. d_compare() uses the
> utf8_strncasecmp() infrastructure, which handles the comparison of
> equivalent, same case, names as well.
>
> For now, negative lookups are not inserted in the dcache, since they
> would need to be invalidated anyway, because we can't trust missing file
> dentries. This is bad for performance but requires some leveraging of
> the vfs layer to fix. We can live without that for now, and so does
> everyone else.
>
> * on-disk data:
>
> Despite using a specific version of the name as the internal
> representation within the dcache, the name stored and fetched from the
> disk is a byte-per-byte match with what the user requested, making this
> implementation 'name-preserving'. i.e. no actual information is lost
> when writing to storage.
>
> DX is supported by modifying the hashes used in +F directories to make
> them case/encoding-aware. The new disk hashes are calculated as the
> hash of the full casefolded string, instead of the string directly.
> This allows us to efficiently search for file names in the htree without
> requiring the user to provide an exact name.
>
> * Dealing with invalid sequences:
>
> By default, when a invalid UTF-8 sequence is identified, ext4 will treat
> it as an opaque byte sequence, ignoring the encoding and reverting to
> the old behavior for that unique file. This means that case-insensitive
> file name lookup will not work only for that file. An optional bit can
> be set in the superblock telling the filesystem code and userspace tools
> to enforce the encoding. When that optional bit is set, any attempt to
> create a file name using an invalid UTF-8 sequence will fail and return
> an error to userspace.
>
> * Normalization algorithm:
>
> The UTF-8 algorithms used to compare strings in f2fs is implemented
> in fs/unicode, and is based on a previous version developed by
> SGI. It implements the Canonical decomposition (NFD) algorithm
> described by the Unicode specification 12.1, or higher, combined with
> the elimination of ignorable code points (NFDi) and full
> case-folding (CF) as documented in fs/unicode/utf8_norm.c.
>
> NFD seems to be the best normalization method for F2FS because:
>
> - It has a lower cost than NFC/NFKC (which requires
> decomposing to NFD as an intermediary step)
> - It doesn't eliminate important semantic meaning like
> compatibility decompositions.
>
> Although:
>
> - This implementation is not completely linguistic accurate, because
> different languages have conflicting rules, which would require the
> specialization of the filesystem to a given locale, which brings all
> sorts of problems for removable media and for users who use more than
> one language.
> """
>
> Signed-off-by: Daniel Rosenberg <drosen@google.com>
> ---
> fs/f2fs/dir.c | 133 ++++++++++++++++++++++++++++++++++++++++++-----
> fs/f2fs/f2fs.h | 18 +++++--
> fs/f2fs/file.c | 10 +++-
> fs/f2fs/hash.c | 34 +++++++++++-
> fs/f2fs/inline.c | 6 +--
> fs/f2fs/inode.c | 4 +-
> fs/f2fs/namei.c | 21 ++++++++
> fs/f2fs/super.c | 5 ++
> 8 files changed, 208 insertions(+), 23 deletions(-)
>
> diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
> index 85a1528f319f2..4d5eea2db1657 100644
> --- a/fs/f2fs/dir.c
> +++ b/fs/f2fs/dir.c
> @@ -8,6 +8,7 @@
> #include <linux/fs.h>
> #include <linux/f2fs_fs.h>
> #include <linux/sched/signal.h>
> +#include <linux/unicode.h>
> #include "f2fs.h"
> #include "node.h"
> #include "acl.h"
> @@ -81,7 +82,8 @@ static unsigned long dir_block_index(unsigned int level,
> return bidx;
> }
>
> -static struct f2fs_dir_entry *find_in_block(struct page *dentry_page,
> +static struct f2fs_dir_entry *find_in_block(struct inode *dir,
> + struct page *dentry_page,
> struct fscrypt_name *fname,
> f2fs_hash_t namehash,
> int *max_slots,
> @@ -94,20 +96,56 @@ static struct f2fs_dir_entry *find_in_block(struct page *dentry_page,
> dentry_blk = (struct f2fs_dentry_block *)page_address(dentry_page);
>
> make_dentry_ptr_block(NULL, &d, dentry_blk);
We can pass dir to make_dentry_ptr_block(dir, ...), then in
f2fs_find_target_dentry() we use d->inode, so that it can avoid one redundant
parameter.
> - de = f2fs_find_target_dentry(fname, namehash, max_slots, &d);
> + de = f2fs_find_target_dentry(dir, fname, namehash, max_slots, &d);
> if (de)
> *res_page = dentry_page;
>
> return de;
> }
>
> -struct f2fs_dir_entry *f2fs_find_target_dentry(struct fscrypt_name *fname,
> - f2fs_hash_t namehash, int *max_slots,
> - struct f2fs_dentry_ptr *d)
> +#ifdef CONFIG_UNICODE
> +/*
> + * Test whether a case-insensitive directory entry matches the filename
> + * being searched for.
> + *
> + * Returns: 0 if the directory entry matches, more than 0 if it
> + * doesn't match or less than zero on error.
> + */
> +int f2fs_ci_compare(const struct inode *parent, const struct qstr *name,
> + const struct qstr *entry)
> +{
> + const struct f2fs_sb_info *sbi = F2FS_SB(parent->i_sb);
> + const struct unicode_map *um = sbi->s_encoding;
> + int ret;
> +
> + ret = utf8_strncasecmp(um, name, entry);
> + if (ret < 0) {
> + /* Handle invalid character sequence as either an error
> + * or as an opaque byte sequence.
> + */
> + if (f2fs_has_strict_mode(sbi))
> + return -EINVAL;
> +
> + if (name->len != entry->len)
> + return 1;
> +
> + return !!memcmp(name->name, entry->name, name->len);
> + }
> +
> + return ret;
> +}
> +#endif
> +
> +struct f2fs_dir_entry *f2fs_find_target_dentry(const struct inode *parent,
> + struct fscrypt_name *fname, f2fs_hash_t namehash,
> + int *max_slots, struct f2fs_dentry_ptr *d)
> {
> struct f2fs_dir_entry *de;
> unsigned long bit_pos = 0;
> int max_len = 0;
> +#ifdef CONFIG_UNICODE
> + struct qstr entry;
> +#endif
>
> if (max_slots)
> *max_slots = 0;
> @@ -119,16 +157,29 @@ struct f2fs_dir_entry *f2fs_find_target_dentry(struct fscrypt_name *fname,
> }
>
> de = &d->dentry[bit_pos];
> +#ifdef CONFIG_UNICODE
> + entry.name = d->filename[bit_pos];
> + entry.len = de->name_len;
> +#endif
>
> if (unlikely(!de->name_len)) {
> bit_pos++;
> continue;
> }
> + if (de->hash_code == namehash) {
> +#ifdef CONFIG_UNICODE
> + if (F2FS_SB(parent->i_sb)->s_encoding &&
> + IS_CASEFOLDED(parent) &&
> + !f2fs_ci_compare(parent,
> + fname->usr_fname, &entry))
> + goto found;
>
> - if (de->hash_code == namehash &&
> - fscrypt_match_name(fname, d->filename[bit_pos],
> - le16_to_cpu(de->name_len)))
> - goto found;
> +#endif
> + if (de->hash_code == namehash &&
It's redundant here.
> + fscrypt_match_name(fname, d->filename[bit_pos],
> + le16_to_cpu(de->name_len)))
> + goto found;
> + }
>
> if (max_slots && max_len > *max_slots)
> *max_slots = max_len;
> @@ -157,7 +208,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
> struct f2fs_dir_entry *de = NULL;
> bool room = false;
> int max_slots;
> - f2fs_hash_t namehash = f2fs_dentry_hash(&name, fname);
> + f2fs_hash_t namehash = f2fs_dentry_hash(dir, &name, fname);
>
> nbucket = dir_buckets(level, F2FS_I(dir)->i_dir_level);
> nblock = bucket_blocks(level);
> @@ -179,8 +230,8 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
> }
> }
>
> - de = find_in_block(dentry_page, fname, namehash, &max_slots,
> - res_page);
> + de = find_in_block(dir, dentry_page, fname, namehash,
> + &max_slots, res_page);
> if (de)
> break;
>
> @@ -246,10 +297,18 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
> struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
> const struct qstr *child, struct page **res_page)
> {
> + struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
> struct f2fs_dir_entry *de = NULL;
> struct fscrypt_name fname;
> int err;
>
> +#ifdef CONFIG_UNICODE
> + if (f2fs_has_strict_mode(sbi) && IS_CASEFOLDED(dir) &&
> + utf8_validate(sbi->s_encoding, child)) {
> + *res_page = ERR_PTR(-EINVAL);
> + return NULL;
> + }
> +#endif
> err = fscrypt_setup_filename(dir, child, 1, &fname);
> if (err) {
> if (err == -ENOENT)
> @@ -504,7 +563,7 @@ int f2fs_add_regular_entry(struct inode *dir, const struct qstr *new_name,
>
> level = 0;
> slots = GET_DENTRY_SLOTS(new_name->len);
> - dentry_hash = f2fs_dentry_hash(new_name, NULL);
> + dentry_hash = f2fs_dentry_hash(dir, new_name, NULL);
>
> current_depth = F2FS_I(dir)->i_current_depth;
> if (F2FS_I(dir)->chash == dentry_hash) {
> @@ -943,3 +1002,51 @@ const struct file_operations f2fs_dir_operations = {
> .compat_ioctl = f2fs_compat_ioctl,
> #endif
> };
> +
> +#ifdef CONFIG_UNICODE
> +static int f2fs_d_compare(const struct dentry *dentry, unsigned int len,
> + const char *str, const struct qstr *name)
> +{
> + struct qstr qstr = {.name = str, .len = len };
> +
> + if (!IS_CASEFOLDED(dentry->d_parent->d_inode)) {
> + if (len != name->len)
> + return -1;
> + return memcmp(str, name, len);
> + }
> +
> + return f2fs_ci_compare(dentry->d_parent->d_inode, name, &qstr);
> +}
> +
> +static int f2fs_d_hash(const struct dentry *dentry, struct qstr *str)
> +{
> + const struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb);
> + const struct unicode_map *um = sbi->s_encoding;
> + unsigned char *norm;
> + int len, ret = 0;
> +
> + if (!IS_CASEFOLDED(dentry->d_inode))
> + return 0;
> +
> + norm = kmalloc(PATH_MAX, GFP_ATOMIC);
f2fs_kmalloc()
> + if (!norm)
> + return -ENOMEM;
> +
> + len = utf8_casefold(um, str, norm, PATH_MAX);
> + if (len < 0) {
> + if (f2fs_has_strict_mode(sbi))
> + ret = -EINVAL;
> + goto out;
> + }
> + str->hash = full_name_hash(dentry, norm, len);
> +out:
> + kfree(norm);
kvfree()
> + return ret;
> +}
> +
> +const struct dentry_operations f2fs_dentry_ops = {
> + .d_hash = f2fs_d_hash,
> + .d_compare = f2fs_d_compare,
> +};
> +#endif
> +
> diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
> index c6c7904572d0d..500906108937c 100644
> --- a/fs/f2fs/f2fs.h
> +++ b/fs/f2fs/f2fs.h
> @@ -2364,10 +2364,12 @@ static inline void f2fs_change_bit(unsigned int nr, char *addr)
> #define F2FS_INDEX_FL 0x00001000 /* hash-indexed directory */
> #define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */
> #define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */
> +#define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */
>
> /* Flags that should be inherited by new inodes from their parent. */
> #define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \
> - F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL)
> + F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
> + F2FS_CASEFOLD_FL)
We need to add one more entry f2fs_fsflags_map[] to map F2FS_CASEFOLD_FL to
FS_CASEFOLD_FL correctly and adapt F2FS_GETTABLE_FS_FL/F2FS_SETTABLE_FS_FL as well.
>
> /* Flags that are appropriate for regular files (all but dir-specific ones). */
> #define F2FS_REG_FLMASK (~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL))
> @@ -2930,11 +2932,16 @@ int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
> bool hot, bool set);
> struct dentry *f2fs_get_parent(struct dentry *child);
>
> +extern int f2fs_ci_compare(const struct inode *parent,
> + const struct qstr *name,
> + const struct qstr *entry);
> +
> /*
> * dir.c
> */
> unsigned char f2fs_get_de_type(struct f2fs_dir_entry *de);
> -struct f2fs_dir_entry *f2fs_find_target_dentry(struct fscrypt_name *fname,
> +struct f2fs_dir_entry *f2fs_find_target_dentry(const struct inode *parent,
> + struct fscrypt_name *fname,
> f2fs_hash_t namehash, int *max_slots,
> struct f2fs_dentry_ptr *d);
> int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
> @@ -2993,8 +3000,8 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
> /*
> * hash.c
> */
> -f2fs_hash_t f2fs_dentry_hash(const struct qstr *name_info,
> - struct fscrypt_name *fname);
> +f2fs_hash_t f2fs_dentry_hash(const struct inode *dir,
> + const struct qstr *name_info, struct fscrypt_name *fname);
>
> /*
> * node.c
> @@ -3437,6 +3444,9 @@ static inline void f2fs_destroy_root_stats(void) { }
> #endif
>
> extern const struct file_operations f2fs_dir_operations;
> +#ifdef CONFIG_UNICODE
> +extern const struct dentry_operations f2fs_dentry_ops;
> +#endif
> extern const struct file_operations f2fs_file_operations;
> extern const struct inode_operations f2fs_file_inode_operations;
> extern const struct address_space_operations f2fs_dblock_aops;
> diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
> index f8d46df8fa9ee..9bdef3aa38eab 100644
> --- a/fs/f2fs/file.c
> +++ b/fs/f2fs/file.c
> @@ -1660,7 +1660,16 @@ static int f2fs_setflags_common(struct inode *inode, u32 iflags, u32 mask)
> return -EPERM;
>
> oldflags = fi->i_flags;
> + if ((iflags ^ oldflags) & F2FS_CASEFOLD_FL) {
> + if (!f2fs_sb_has_casefold(F2FS_I_SB(inode)))
> + return -EOPNOTSUPP;
> +
> + if (!S_ISDIR(inode->i_mode))
> + return -ENOTDIR;
>
> + if (!f2fs_empty_dir(inode))
> + return -ENOTEMPTY;
> + }
> if ((iflags ^ oldflags) & (F2FS_APPEND_FL | F2FS_IMMUTABLE_FL))
> if (!capable(CAP_LINUX_IMMUTABLE))
> return -EPERM;
> @@ -1671,7 +1680,6 @@ static int f2fs_setflags_common(struct inode *inode, u32 iflags, u32 mask)
> set_inode_flag(inode, FI_PROJ_INHERIT);
> else
> clear_inode_flag(inode, FI_PROJ_INHERIT);
> -
Unneeded removal...
> inode->i_ctime = current_time(inode);
> f2fs_set_inode_flags(inode);
> f2fs_mark_inode_dirty_sync(inode, true);
> diff --git a/fs/f2fs/hash.c b/fs/f2fs/hash.c
> index cc82f142f811f..f5b8e02bde049 100644
> --- a/fs/f2fs/hash.c
> +++ b/fs/f2fs/hash.c
> @@ -14,6 +14,7 @@
> #include <linux/f2fs_fs.h>
> #include <linux/cryptohash.h>
> #include <linux/pagemap.h>
> +#include <linux/unicode.h>
>
> #include "f2fs.h"
>
> @@ -67,7 +68,7 @@ static void str2hashbuf(const unsigned char *msg, size_t len,
> *buf++ = pad;
> }
>
> -f2fs_hash_t f2fs_dentry_hash(const struct qstr *name_info,
> +static f2fs_hash_t __f2fs_dentry_hash(const struct qstr *name_info,
> struct fscrypt_name *fname)
> {
> __u32 hash;
> @@ -103,3 +104,34 @@ f2fs_hash_t f2fs_dentry_hash(const struct qstr *name_info,
> f2fs_hash = cpu_to_le32(hash & ~F2FS_HASH_COL_BIT);
> return f2fs_hash;
> }
> +
> +f2fs_hash_t f2fs_dentry_hash(const struct inode *dir,
> + const struct qstr *name_info, struct fscrypt_name *fname)
> +{
> +#ifdef CONFIG_UNICODE
> + const struct unicode_map *um = F2FS_SB(dir->i_sb)->s_encoding;
> + int r, dlen;
> + unsigned char *buff;
> + struct qstr *folded;
> +
> + if (name_info->len && IS_CASEFOLDED(dir)) {
> + buff = kzalloc(sizeof(char) * PATH_MAX, GFP_KERNEL);
f2fs_kzalloc()
> + if (!buff)
> + return -ENOMEM;
> +
> + dlen = utf8_casefold(um, name_info, buff, PATH_MAX);
> + if (dlen < 0) {
> + kfree(buff);
> + goto opaque_seq;
> + }
> + folded->name = buff;
> + folded->len = dlen;
> + r = __f2fs_dentry_hash(folded, fname);
> +
> + kfree(buff);
kvfree()
> + return r;
> + }
> +opaque_seq:
> +#endif
> + return __f2fs_dentry_hash(name_info, fname);
> +}
> diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
> index 3613efca8c00c..7cff67af4fadb 100644
> --- a/fs/f2fs/inline.c
> +++ b/fs/f2fs/inline.c
> @@ -320,12 +320,12 @@ struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
> return NULL;
> }
>
> - namehash = f2fs_dentry_hash(&name, fname);
> + namehash = f2fs_dentry_hash(dir, &name, fname);
>
> inline_dentry = inline_data_addr(dir, ipage);
>
> make_dentry_ptr_inline(dir, &d, inline_dentry);
> - de = f2fs_find_target_dentry(fname, namehash, NULL, &d);
> + de = f2fs_find_target_dentry(dir, fname, namehash, NULL, &d);
We don't need to pass @dir, try using d->inode in f2fs_find_target_dentry()
> unlock_page(ipage);
> if (de)
> *res_page = ipage;
> @@ -580,7 +580,7 @@ int f2fs_add_inline_entry(struct inode *dir, const struct qstr *new_name,
>
> f2fs_wait_on_page_writeback(ipage, NODE, true, true);
>
> - name_hash = f2fs_dentry_hash(new_name, NULL);
> + name_hash = f2fs_dentry_hash(dir, new_name, NULL);
> f2fs_update_dentry(ino, mode, &d, new_name, name_hash, bit_pos);
>
> set_page_dirty(ipage);
> diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
> index a33d7a849b2df..9a1f0d6616577 100644
> --- a/fs/f2fs/inode.c
> +++ b/fs/f2fs/inode.c
> @@ -46,9 +46,11 @@ void f2fs_set_inode_flags(struct inode *inode)
> new_fl |= S_DIRSYNC;
> if (file_is_encrypt(inode))
> new_fl |= S_ENCRYPTED;
> + if (flags & F2FS_CASEFOLD_FL)
> + new_fl |= S_CASEFOLD;
> inode_set_flags(inode, new_fl,
> S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|
> - S_ENCRYPTED);
> + S_ENCRYPTED|S_CASEFOLD);
> }
>
> static void __get_inode_rdev(struct inode *inode, struct f2fs_inode *ri)
> diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
> index c5b99042e6f2b..727de2f8620f2 100644
> --- a/fs/f2fs/namei.c
> +++ b/fs/f2fs/namei.c
> @@ -489,6 +489,17 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
> goto out_iput;
> }
> out_splice:
> +#ifdef CONFIG_UNICODE
> + if (!inode && IS_CASEFOLDED(dir)) {
> + /* Eventually we want to call d_add_ci(dentry, NULL)
> + * for negative dentries in the encoding case as
> + * well. For now, prevent the negative dentry
> + * from being cached.
> + */
> + trace_f2fs_lookup_end(dir, dentry, ino, err);
> + return NULL;
> + }
> +#endif
> new = d_splice_alias(inode, dentry);
> err = PTR_ERR_OR_ZERO(new);
> trace_f2fs_lookup_end(dir, dentry, ino, err);
> @@ -537,6 +548,16 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry)
> goto fail;
> }
> f2fs_delete_entry(de, page, dir, inode);
> +#ifdef CONFIG_UNICODE
> + /* VFS negative dentries are incompatible with Encoding and
> + * Case-insensitiveness. Eventually we'll want avoid
> + * invalidating the dentries here, alongside with returning the
> + * negative dentries at f2fs_lookup(), when it is better
> + * supported by the VFS for the CI case.
> + */
> + if (IS_CASEFOLDED(dir))
> + d_invalidate(dentry);
> +#endif
> f2fs_unlock_op(sbi);
>
> if (IS_DIRSYNC(dir))
> diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
> index 7927071ef5e95..edc8482a43604 100644
> --- a/fs/f2fs/super.c
> +++ b/fs/f2fs/super.c
> @@ -3397,6 +3397,11 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
> goto free_node_inode;
> }
>
> +#ifdef CONFIG_UNICODE
> + if (sbi->s_encoding)
> + sb->s_d_op = &f2fs_dentry_ops;
> +#endif
> +
> sb->s_root = d_make_root(root); /* allocate root dentry */
> if (!sb->s_root) {
> err = -ENOMEM;
>
^ permalink raw reply
* Re: [PATCH v2 1/2] f2fs: include charset encoding information in the superblock
From: Chao Yu @ 2019-07-17 9:39 UTC (permalink / raw)
To: Daniel Rosenberg, Jaegeuk Kim, Jonathan Corbet, linux-f2fs-devel
Cc: linux-kernel, linux-doc, linux-fsdevel, kernel-team
In-Reply-To: <20190717031408.114104-2-drosen@google.com>
On 2019/7/17 11:14, Daniel Rosenberg wrote:
> Add charset encoding to f2fs to support casefolding. It is modeled after
> the same feature introduced in commit c83ad55eaa91 ("ext4: include charset
> encoding information in the superblock")
>
> Currently this is not compatible with encryption, similar to the current
> ext4 imlpementation. This will change in the future.
>
>>From the ext4 patch:
> """
> The s_encoding field stores a magic number indicating the encoding
> format and version used globally by file and directory names in the
> filesystem. The s_encoding_flags defines policies for using the charset
> encoding, like how to handle invalid sequences. The magic number is
> mapped to the exact charset table, but the mapping is specific to ext4.
> Since we don't have any commitment to support old encodings, the only
> encoding I am supporting right now is utf8-12.1.0.
>
> The current implementation prevents the user from enabling encoding and
> per-directory encryption on the same filesystem at the same time. The
> incompatibility between these features lies in how we do efficient
> directory searches when we cannot be sure the encryption of the user
> provided fname will match the actual hash stored in the disk without
> decrypting every directory entry, because of normalization cases. My
> quickest solution is to simply block the concurrent use of these
> features for now, and enable it later, once we have a better solution.
> """
>
> Signed-off-by: Daniel Rosenberg <drosen@google.com>
> ---
> fs/f2fs/f2fs.h | 6 +++
> fs/f2fs/super.c | 81 +++++++++++++++++++++++++++++++++++++++++
> include/linux/f2fs_fs.h | 9 ++++-
> 3 files changed, 95 insertions(+), 1 deletion(-)
>
> diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
> index 17382da7f0bd9..c6c7904572d0d 100644
> --- a/fs/f2fs/f2fs.h
> +++ b/fs/f2fs/f2fs.h
> @@ -153,6 +153,7 @@ struct f2fs_mount_info {
> #define F2FS_FEATURE_LOST_FOUND 0x0200
> #define F2FS_FEATURE_VERITY 0x0400 /* reserved */
> #define F2FS_FEATURE_SB_CHKSUM 0x0800
> +#define F2FS_FEATURE_CASEFOLD 0x1000
>
> #define __F2FS_HAS_FEATURE(raw_super, mask) \
> ((raw_super->feature & cpu_to_le32(mask)) != 0)
> @@ -1169,6 +1170,10 @@ struct f2fs_sb_info {
> int valid_super_block; /* valid super block no */
> unsigned long s_flag; /* flags for sbi */
> struct mutex writepages; /* mutex for writepages() */
> +#ifdef CONFIG_UNICODE
> + struct unicode_map *s_encoding;
> + __u16 s_encoding_flags;
> +#endif
>
> #ifdef CONFIG_BLK_DEV_ZONED
> unsigned int blocks_per_blkz; /* F2FS blocks per zone */
> @@ -3562,6 +3567,7 @@ F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO);
> F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME);
> F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND);
> F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
> +F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
It needs to change sysfs.c like we did for other features, you can refer to
d440c52d3151 ("f2fs: support superblock checksum").
>
> #ifdef CONFIG_BLK_DEV_ZONED
> static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
> diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
> index 6de6cda440315..7927071ef5e95 100644
> --- a/fs/f2fs/super.c
> +++ b/fs/f2fs/super.c
> @@ -23,6 +23,7 @@
> #include <linux/f2fs_fs.h>
> #include <linux/sysfs.h>
> #include <linux/quota.h>
> +#include <linux/unicode.h>
>
> #include "f2fs.h"
> #include "node.h"
> @@ -222,6 +223,36 @@ void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...)
> va_end(args);
> }
>
> +#ifdef CONFIG_UNICODE
> +static const struct f2fs_sb_encodings {
> + __u16 magic;
> + char *name;
> + char *version;
> +} f2fs_sb_encoding_map[] = {
> + {F2FS_ENC_UTF8_12_1, "utf8", "12.1.0"},
> +};
> +
> +static int f2fs_sb_read_encoding(const struct f2fs_super_block *sb,
> + const struct f2fs_sb_encodings **encoding,
> + __u16 *flags)
> +{
> + __u16 magic = le16_to_cpu(sb->s_encoding);
> + int i;
> +
> + for (i = 0; i < ARRAY_SIZE(f2fs_sb_encoding_map); i++)
> + if (magic == f2fs_sb_encoding_map[i].magic)
> + break;
> +
> + if (i >= ARRAY_SIZE(f2fs_sb_encoding_map))
> + return -EINVAL;
> +
> + *encoding = &f2fs_sb_encoding_map[i];
> + *flags = le16_to_cpu(sb->s_encoding_flags);
> +
> + return 0;
> +}
> +#endif
> +
> static inline void limit_reserve_root(struct f2fs_sb_info *sbi)
> {
> block_t limit = min((sbi->user_block_count << 1) / 1000,
> @@ -798,6 +829,13 @@ static int parse_options(struct super_block *sb, char *options)
> return -EINVAL;
> }
> #endif
> +#ifndef CONFIG_UNICODE
> + if (f2fs_sb_has_casefold(sbi)) {
> + f2fs_err(sbi,
> + "Filesystem with casefold feature cannot be mounted without CONFIG_UNICODE");
> + return -EINVAL;
> + }
> +#endif
>
> if (F2FS_IO_SIZE_BITS(sbi) && !test_opt(sbi, LFS)) {
> f2fs_err(sbi, "Should set mode=lfs with %uKB-sized IO",
> @@ -1089,6 +1127,9 @@ static void f2fs_put_super(struct super_block *sb)
> destroy_percpu_info(sbi);
> for (i = 0; i < NR_PAGE_TYPE; i++)
> kvfree(sbi->write_io[i]);
> +#ifdef CONFIG_UNICODE
> + utf8_unload(sbi->s_encoding);
> +#endif
> kvfree(sbi);
> }
>
> @@ -3126,6 +3167,42 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
> sb->s_maxbytes = sbi->max_file_blocks <<
> le32_to_cpu(raw_super->log_blocksize);
> sb->s_max_links = F2FS_LINK_MAX;
> +#ifdef CONFIG_UNICODE
> + if (f2fs_sb_has_casefold(sbi) && !sbi->s_encoding) {
> + const struct f2fs_sb_encodings *encoding_info;
> + struct unicode_map *encoding;
> + __u16 encoding_flags;
> +
> + if (f2fs_sb_has_encrypt(sbi)) {
> + f2fs_err(sbi,
> + "Can't mount with encoding and encryption");
Should set error number.
> + goto free_options;
> + }
> +
> + if (f2fs_sb_read_encoding(raw_super, &encoding_info,
> + &encoding_flags)) {
> + f2fs_err(sbi,
> + "Encoding requested by superblock is unknown");
Ditto
> + goto free_options;
> + }
> +
> + encoding = utf8_load(encoding_info->version);
> + if (IS_ERR(encoding)) {
> + f2fs_err(sbi,
> + "can't mount with superblock charset: %s-%s "
> + "not supported by the kernel. flags: 0x%x.",
> + encoding_info->name, encoding_info->version,
> + encoding_flags);
Ditto
> + goto free_options;
> + }
> + f2fs_info(sbi, "Using encoding defined by superblock: "
> + "%s-%s with flags 0x%hx", encoding_info->name,
> + encoding_info->version?:"\b", encoding_flags);
> +
> + sbi->s_encoding = encoding;
> + sbi->s_encoding_flags = encoding_flags;
> + }
> +#endif
How about wrapping these codes into function?
Thanks,
>
> #ifdef CONFIG_QUOTA
> sb->dq_op = &f2fs_quota_operations;
> @@ -3477,6 +3554,10 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
> free_bio_info:
> for (i = 0; i < NR_PAGE_TYPE; i++)
> kvfree(sbi->write_io[i]);
> +
> +#ifdef CONFIG_UNICODE
> + utf8_unload(sbi->s_encoding);
> +#endif
> free_options:
> #ifdef CONFIG_QUOTA
> for (i = 0; i < MAXQUOTAS; i++)
> diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h
> index 65559900d4d76..b7c9c7f721339 100644
> --- a/include/linux/f2fs_fs.h
> +++ b/include/linux/f2fs_fs.h
> @@ -36,6 +36,11 @@
>
> #define F2FS_MAX_QUOTAS 3
>
> +#define F2FS_ENC_UTF8_12_1 1
> +#define F2FS_ENC_STRICT_MODE_FL (1 << 0)
> +#define f2fs_has_strict_mode(sbi) \
> + (sbi->s_encoding_flags & F2FS_ENC_STRICT_MODE_FL)
> +
> #define F2FS_IO_SIZE(sbi) (1 << F2FS_OPTION(sbi).write_io_size_bits) /* Blocks */
> #define F2FS_IO_SIZE_KB(sbi) (1 << (F2FS_OPTION(sbi).write_io_size_bits + 2)) /* KB */
> #define F2FS_IO_SIZE_BYTES(sbi) (1 << (F2FS_OPTION(sbi).write_io_size_bits + 12)) /* B */
> @@ -109,7 +114,9 @@ struct f2fs_super_block {
> struct f2fs_device devs[MAX_DEVICES]; /* device list */
> __le32 qf_ino[F2FS_MAX_QUOTAS]; /* quota inode numbers */
> __u8 hot_ext_count; /* # of hot file extension */
> - __u8 reserved[310]; /* valid reserved region */
> + __le16 s_encoding; /* Filename charset encoding */
> + __le16 s_encoding_flags; /* Filename charset encoding flags */
> + __u8 reserved[306]; /* valid reserved region */
> __le32 crc; /* checksum of superblock */
> } __packed;
>
>
^ permalink raw reply
* [PATCH v3 00/12] kbuild: create *.mod with directory path and remove MODVERDIR
From: Masahiro Yamada @ 2019-07-17 6:17 UTC (permalink / raw)
To: linux-kbuild
Cc: Joe Lawrence, Masahiro Yamada, James E.J. Bottomley,
Jonathan Corbet, Martin K. Petersen, Michal Marek, Shuah Khan,
Thomas Renninger, linux-doc, linux-kernel, linux-pm, linux-scsi
This series kills the long standing MODVERDIR.
Since MODVERDIR has a flat structure, it cannot avoid a race
condition when somebody introduces a module name conflict.
Kbuild now reads modules.order to get the list of all modules.
The post-processing/installation stages will be more robust
and simpler.
Masahiro Yamada (12):
kbuild: do not create empty modules.order in the prepare stage
kbuild: get rid of kernel/ prefix from in-tree modules.{order,builtin}
kbuild: remove duplication from modules.order in sub-directories
scsi: remove pointless $(MODVERDIR)/$(obj)/53c700.ver
kbuild: modinst: read modules.order instead of $(MODVERDIR)/*.mod
kbuild: modsign: read modules.order instead of $(MODVERDIR)/*.mod
kbuild: modpost: read modules.order instead of $(MODVERDIR)/*.mod
kbuild: export_report: read modules.order instead of
.tmp_versions/*.mod
kbuild: create *.mod with full directory path and remove MODVERDIR
kbuild: remove the first line of *.mod files
kbuild: remove 'prepare1' target
kbuild: split out *.mod out of {single,multi}-used-m rules
.gitignore | 1 +
Documentation/dontdiff | 1 +
Makefile | 36 ++++++-------------
drivers/scsi/Makefile | 2 +-
lib/Kconfig.debug | 12 +------
scripts/Makefile.build | 40 +++++++++-------------
scripts/Makefile.modbuiltin | 2 +-
scripts/Makefile.modinst | 5 +--
scripts/Makefile.modpost | 19 +++++-----
scripts/Makefile.modsign | 3 +-
scripts/adjust_autoksyms.sh | 14 +++-----
scripts/export_report.pl | 11 +++---
scripts/mod/sumversion.c | 23 +++----------
scripts/modules-check.sh | 2 +-
scripts/package/mkspec | 2 +-
tools/power/cpupower/debug/kernel/Makefile | 4 +--
16 files changed, 62 insertions(+), 115 deletions(-)
--
2.17.1
^ permalink raw reply
* [PATCH v3 09/12] kbuild: create *.mod with full directory path and remove MODVERDIR
From: Masahiro Yamada @ 2019-07-17 6:17 UTC (permalink / raw)
To: linux-kbuild
Cc: Joe Lawrence, Masahiro Yamada, Jonathan Corbet, Michal Marek,
Shuah Khan, Thomas Renninger, linux-doc, linux-kernel, linux-pm
In-Reply-To: <20190717061800.10018-1-yamada.masahiro@socionext.com>
While descending directories, Kbuild produces objects for modules,
but do not link final *.ko files; it is done in the modpost.
To keep track of modules, Kbuild creates a *.mod file in $(MODVERDIR)
for every module it is building. Some post-processing steps read the
necessary information from *.mod files. This avoids descending into
directories again. This mechanism was introduced in 2003 or so.
Later, commit 551559e13af1 ("kbuild: implement modules.order") added
modules.order. So, we can simply read it out to know all the modules
with directory paths. This is easier than parsing the first line of
*.mod files.
$(MODVERDIR) has a flat directory structure, that is, *.mod files
are named only with base names. This is based on the assumption that
the module name is unique across the tree. This assumption is really
fragile.
Stephen Rothwell reported a race condition caused by a module name
conflict:
https://lkml.org/lkml/2019/5/13/991
In parallel building, two different threads could write to the same
$(MODVERDIR)/*.mod simultaneously.
Non-unique module names are the source of all kind of troubles, hence
commit 3a48a91901c5 ("kbuild: check uniqueness of module names")
introduced a new checker script.
However, it is still fragile in the build system point of view because
this race happens before scripts/modules-check.sh is invoked. If it
happens again, the modpost will emit unclear error messages.
To fix this issue completely, create *.mod with full directory path
so that two threads never attempt to write to the same file.
$(MODVERDIR) is no longer needed.
Since modules with directory paths are listed in modules.order, Kbuild
is still able to find *.mod files without additional descending.
I also killed cmd_secanalysis; scripts/mod/sumversion.c computes MD4 hash
for modules with MODULE_VERSION(). When CONFIG_DEBUG_SECTION_MISMATCH=y,
it occurs not only in the modpost stage, but also during directory
descending, where sumversion.c may parse stale *.mod files. It would emit
'No such file or directory' warning when an object consisting a module is
renamed, or when a single-obj module is turned into a multi-obj module or
vice versa.
Signed-off-by: Masahiro Yamada <yamada.masahiro@socionext.com>
Acked-by: Nicolas Pitre <nico@fluxnic.net>
---
Changes in v3:
- Fix build error of allnoconfig
- Remove cmd_secanalysis
- Fix up comment in scripts/adjust_autoksyms.sh
- Fix up tools/power/cpupower/debug/kernel/Makefile
Changes in v2:
- Remove -r of xargs, which is a GNU extension
- Add '--' for extra safety
.gitignore | 1 +
Documentation/dontdiff | 1 +
Makefile | 20 +++-----------------
lib/Kconfig.debug | 12 +-----------
scripts/Makefile.build | 15 +++------------
scripts/Makefile.modpost | 4 ++--
scripts/adjust_autoksyms.sh | 14 +++++---------
scripts/mod/sumversion.c | 16 +++-------------
scripts/package/mkspec | 2 +-
tools/power/cpupower/debug/kernel/Makefile | 4 ++--
10 files changed, 22 insertions(+), 67 deletions(-)
diff --git a/.gitignore b/.gitignore
index 7587ef56b92d..8f5422cba6e2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@
*.lz4
*.lzma
*.lzo
+*.mod
*.mod.c
*.o
*.o.*
diff --git a/Documentation/dontdiff b/Documentation/dontdiff
index 5eba889ea84d..9f4392876099 100644
--- a/Documentation/dontdiff
+++ b/Documentation/dontdiff
@@ -30,6 +30,7 @@
*.lzo
*.mo
*.moc
+*.mod
*.mod.c
*.o
*.o.*
diff --git a/Makefile b/Makefile
index 396cd5e525d1..9ad9f8d1130d 100644
--- a/Makefile
+++ b/Makefile
@@ -486,11 +486,6 @@ export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE
export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL
export KBUILD_ARFLAGS
-# When compiling out-of-tree modules, put MODVERDIR in the module
-# tree rather than in the kernel tree. The kernel tree might
-# even be read-only.
-export MODVERDIR := $(if $(KBUILD_EXTMOD),$(firstword $(KBUILD_EXTMOD))/).tmp_versions
-
# Files to ignore in find ... statements
export RCS_FIND_IGNORE := \( -name SCCS -o -name BitKeeper -o -name .svn -o \
@@ -1029,8 +1024,8 @@ vmlinux-deps := $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)
# Recurse until adjust_autoksyms.sh is satisfied
PHONY += autoksyms_recursive
-autoksyms_recursive: $(vmlinux-deps)
ifdef CONFIG_TRIM_UNUSED_KSYMS
+autoksyms_recursive: $(vmlinux-deps) modules.order
$(Q)$(CONFIG_SHELL) $(srctree)/scripts/adjust_autoksyms.sh \
"$(MAKE) -f $(srctree)/Makefile vmlinux"
endif
@@ -1113,7 +1108,6 @@ endif
prepare1: prepare3 outputmakefile asm-generic $(version_h) $(autoksyms_h) \
include/generated/utsrelease.h
- $(cmd_crmodverdir)
archprepare: archheaders archscripts prepare1 scripts
@@ -1371,7 +1365,7 @@ endif # CONFIG_MODULES
# make distclean Remove editor backup files, patch leftover files and the like
# Directories & files removed with 'make clean'
-CLEAN_DIRS += $(MODVERDIR) include/ksym
+CLEAN_DIRS += include/ksym
CLEAN_FILES += modules.builtin.modinfo
# Directories & files removed with 'make mrproper'
@@ -1641,7 +1635,6 @@ PHONY += $(clean-dirs) clean
$(clean-dirs):
$(Q)$(MAKE) $(clean)=$(patsubst _clean_%,%,$@)
-clean: rm-dirs := $(MODVERDIR)
clean: rm-files := $(KBUILD_EXTMOD)/Module.symvers
PHONY += help
@@ -1655,8 +1648,6 @@ help:
@echo ''
PHONY += prepare
-prepare:
- $(cmd_crmodverdir)
endif # KBUILD_EXTMOD
clean: $(clean-dirs)
@@ -1667,7 +1658,7 @@ clean: $(clean-dirs)
-o -name '*.ko.*' \
-o -name '*.dtb' -o -name '*.dtb.S' -o -name '*.dt.yaml' \
-o -name '*.dwo' -o -name '*.lst' \
- -o -name '*.su' \
+ -o -name '*.su' -o -name '*.mod' \
-o -name '.*.d' -o -name '.*.tmp' -o -name '*.mod.c' \
-o -name '*.lex.c' -o -name '*.tab.[ch]' \
-o -name '*.asn1.[ch]' \
@@ -1796,11 +1787,6 @@ quiet_cmd_depmod = DEPMOD $(KERNELRELEASE)
cmd_depmod = $(CONFIG_SHELL) $(srctree)/scripts/depmod.sh $(DEPMOD) \
$(KERNELRELEASE)
-# Create temporary dir for module support files
-# clean it up only when building all modules
-cmd_crmodverdir = $(Q)mkdir -p $(MODVERDIR) \
- $(if $(KBUILD_MODULES),; rm -f $(MODVERDIR)/*)
-
# read saved command lines for existing targets
existing-targets := $(wildcard $(sort $(targets)))
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 4ac4ca21a30a..cde5675340ba 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -353,23 +353,13 @@ config DEBUG_SECTION_MISMATCH
which results in the code/data being placed in specific sections.
The section mismatch analysis is always performed after a full
kernel build, and enabling this option causes the following
- additional steps to occur:
+ additional step to occur:
- Add the option -fno-inline-functions-called-once to gcc commands.
When inlining a function annotated with __init in a non-init
function, we would lose the section information and thus
the analysis would not catch the illegal reference.
This option tells gcc to inline less (but it does result in
a larger kernel).
- - Run the section mismatch analysis for each module/built-in.a file.
- When we run the section mismatch analysis on vmlinux.o, we
- lose valuable information about where the mismatch was
- introduced.
- Running the analysis for each module/built-in.a file
- tells where the mismatch happens much closer to the
- source. The drawback is that the same mismatch is
- reported at least twice.
- - Enable verbose reporting from modpost in order to help resolve
- the section mismatches that are reported.
config SECTION_MISMATCH_WARN_ONLY
bool "Make section mismatch errors non-fatal"
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index be32a3752de4..c6dfcc028f56 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -67,8 +67,6 @@ ifeq ($(CONFIG_MODULES)$(need-modorder),y1)
modorder-target := $(obj)/modules.order
endif
-# We keep a list of all modules in $(MODVERDIR)
-
__build: $(if $(KBUILD_BUILTIN),$(builtin-target) $(lib-target) $(extra-y)) \
$(if $(KBUILD_MODULES),$(obj-m) $(modorder-target)) \
$(subdir-ym) $(always)
@@ -87,11 +85,6 @@ ifneq ($(KBUILD_ENABLE_EXTRA_GCC_CHECKS),)
cmd_checkdoc = $(srctree)/scripts/kernel-doc -none $<
endif
-# Do section mismatch analysis for each module/built-in.a
-ifdef CONFIG_DEBUG_SECTION_MISMATCH
- cmd_secanalysis = ; scripts/mod/modpost $@
-endif
-
# Compile C sources (.c)
# ---------------------------------------------------------------------------
@@ -278,13 +271,11 @@ $(obj)/%.o: $(src)/%.c $(recordmcount_source) $(objtool_dep) FORCE
$(call cmd,force_checksrc)
$(call if_changed_rule,cc_o_c)
-# Single-part modules are special since we need to mark them in $(MODVERDIR)
-
$(single-used-m): $(obj)/%.o: $(src)/%.c $(recordmcount_source) $(objtool_dep) FORCE
$(call cmd,force_checksrc)
$(call if_changed_rule,cc_o_c)
@{ echo $(@:.o=.ko); echo $@; \
- $(cmd_undef_syms); } > $(MODVERDIR)/$(@F:.o=.mod)
+ $(cmd_undef_syms); } > $(patsubst %.o,%.mod,$@)
quiet_cmd_cc_lst_c = MKLST $@
cmd_cc_lst_c = $(CC) $(c_flags) -g -c -o $*.o $< && \
@@ -461,12 +452,12 @@ endif
# module is turned into a multi object module, $^ will contain header file
# dependencies recorded in the .*.cmd file.
quiet_cmd_link_multi-m = LD [M] $@
-cmd_link_multi-m = $(LD) $(ld_flags) -r -o $@ $(filter %.o,$^) $(cmd_secanalysis)
+ cmd_link_multi-m = $(LD) $(ld_flags) -r -o $@ $(filter %.o,$^)
$(multi-used-m): FORCE
$(call if_changed,link_multi-m)
@{ echo $(@:.o=.ko); echo $(filter %.o,$^); \
- $(cmd_undef_syms); } > $(MODVERDIR)/$(@F:.o=.mod)
+ $(cmd_undef_syms); } > $(patsubst %.o,%.mod,$@)
$(call multi_depend, $(multi-used-m), .o, -objs -y -m)
targets += $(multi-used-m)
diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost
index 5841508ffca9..6b19c1a4eae5 100644
--- a/scripts/Makefile.modpost
+++ b/scripts/Makefile.modpost
@@ -6,8 +6,8 @@
# Stage one of module building created the following:
# a) The individual .o files used for the module
# b) A <module>.o file which is the .o files above linked together
-# c) A <module>.mod file in $(MODVERDIR)/, listing the name of the
-# the preliminary <module>.o file, plus all .o files
+# c) A <module>.mod file, listing the name of the preliminary <module>.o file,
+# plus all .o files
# d) modules.order, which lists all the modules
# Stage 2 is handled by this file and does the following
diff --git a/scripts/adjust_autoksyms.sh b/scripts/adjust_autoksyms.sh
index aab4e299d7a2..2e4a7320bfb4 100755
--- a/scripts/adjust_autoksyms.sh
+++ b/scripts/adjust_autoksyms.sh
@@ -8,8 +8,7 @@
#
# Create/update the include/generated/autoksyms.h file from the list
-# of all module's needed symbols as recorded on the third line of
-# .tmp_versions/*.mod files.
+# of all module's needed symbols as recorded on the third line of *.mod files.
#
# For each symbol being added or removed, the corresponding dependency
# file's timestamp is updated to force a rebuild of the affected source
@@ -47,13 +46,10 @@ cat > "$new_ksyms_file" << EOT
*/
EOT
-[ "$(ls -A "$MODVERDIR")" ] &&
-for mod in "$MODVERDIR"/*.mod; do
- sed -n -e '3{s/ /\n/g;/^$/!p;}' "$mod"
-done | sort -u |
-while read sym; do
- echo "#define __KSYM_${sym} 1"
-done >> "$new_ksyms_file"
+sed 's/ko$/mod/' modules.order |
+xargs -n1 sed -n -e '3{s/ /\n/g;/^$/!p;}' -- |
+sort -u |
+sed -e 's/\(.*\)/#define __KSYM_\1 1/' >> "$new_ksyms_file"
# Special case for modversions (see modpost.c)
if [ -n "$CONFIG_MODVERSIONS" ]; then
diff --git a/scripts/mod/sumversion.c b/scripts/mod/sumversion.c
index 0f6dcb4011a8..166f3fa247a9 100644
--- a/scripts/mod/sumversion.c
+++ b/scripts/mod/sumversion.c
@@ -396,21 +396,11 @@ void get_src_version(const char *modname, char sum[], unsigned sumlen)
unsigned long len;
struct md4_ctx md;
char *sources, *end, *fname;
- const char *basename;
char filelist[PATH_MAX + 1];
- char *modverdir = getenv("MODVERDIR");
- if (!modverdir)
- modverdir = ".";
-
- /* Source files for module are in .tmp_versions/modname.mod,
- after the first line. */
- if (strrchr(modname, '/'))
- basename = strrchr(modname, '/') + 1;
- else
- basename = modname;
- snprintf(filelist, sizeof(filelist), "%s/%.*s.mod", modverdir,
- (int) strlen(basename) - 2, basename);
+ /* objects for a module are listed in the second line of *.mod file. */
+ snprintf(filelist, sizeof(filelist), "%.*smod",
+ (int)strlen(modname) - 1, modname);
file = grab_file(filelist, &len);
if (!file)
diff --git a/scripts/package/mkspec b/scripts/package/mkspec
index 2d29df4a0a53..8640c278f1aa 100755
--- a/scripts/package/mkspec
+++ b/scripts/package/mkspec
@@ -29,7 +29,7 @@ fi
PROVIDES="$PROVIDES kernel-$KERNELRELEASE"
__KERNELRELEASE=$(echo $KERNELRELEASE | sed -e "s/-/_/g")
-EXCLUDES="$RCS_TAR_IGNORE --exclude=.tmp_versions --exclude=*vmlinux* \
+EXCLUDES="$RCS_TAR_IGNORE --exclude=*vmlinux* --exclude=*.mod \
--exclude=*.o --exclude=*.ko --exclude=*.cmd --exclude=Documentation \
--exclude=.config.old --exclude=.missing-syscalls.d --exclude=*.s"
diff --git a/tools/power/cpupower/debug/kernel/Makefile b/tools/power/cpupower/debug/kernel/Makefile
index c23e5a6ceb7e..7b5c43684be1 100644
--- a/tools/power/cpupower/debug/kernel/Makefile
+++ b/tools/power/cpupower/debug/kernel/Makefile
@@ -12,8 +12,8 @@ default:
$(MAKE) -C $(KDIR) M=$(CURDIR)
clean:
- - rm -rf *.o *.ko .tmp-versions .*.cmd .*.mod.* *.mod.c
- - rm -rf .tmp_versions* Module.symvers modules.order
+ - rm -rf *.o *.ko .*.cmd .*.mod.* *.mod.c
+ - rm -rf Module.symvers modules.order
install: default
install -d $(KMISC)
--
2.17.1
^ permalink raw reply related
* Re: [PATCH v2 08/11] kbuild: create *.mod with full directory path and remove MODVERDIR
From: Masahiro Yamada @ 2019-07-17 5:21 UTC (permalink / raw)
To: Joe Lawrence
Cc: Linux Kbuild mailing list, Sam Ravnborg, Nicolas Pitre,
open list:DOCUMENTATION, Jonathan Corbet,
Linux Kernel Mailing List, Michal Marek
In-Reply-To: <20190716214023.GA15159@redhat.com>
Hi Joe
On Wed, Jul 17, 2019 at 6:40 AM Joe Lawrence <joe.lawrence@redhat.com> wrote:
>
> On Thu, Jul 11, 2019 at 02:44:31PM +0900, Masahiro Yamada wrote:
> > While descending directories, Kbuild produces objects for modules,
> > but do not link final *.ko files; it is done in the modpost.
> >
> > To keep track of modules, Kbuild creates a *.mod file in $(MODVERDIR)
> > for every module it is building. Some post-processing steps read the
> > necessary information from *.mod files. This avoids descending into
> > directories again. This mechanism was introduced in 2003 or so.
> >
> > Later, commit 551559e13af1 ("kbuild: implement modules.order") added
> > modules.order. So, we can simply read it out to know all the modules
> > with directory paths. This is easier than parsing the first line of
> > *.mod files.
> >
> > $(MODVERDIR) has a flat directory structure, that is, *.mod files
> > are named only with base names. This is based on the assumption that
> > the module name is unique across the tree. This assumption is really
> > fragile.
> >
> > Stephen Rothwell reported a race condition caused by a module name
> > conflict:
> >
> > https://lkml.org/lkml/2019/5/13/991
> >
> > In parallel building, two different threads could write to the same
> > $(MODVERDIR)/*.mod simultaneously.
> >
> > Non-unique module names are the source of all kind of troubles, hence
> > commit 3a48a91901c5 ("kbuild: check uniqueness of module names")
> > introduced a new checker script.
> >
> > However, it is still fragile in the build system point of view because
> > this race happens before scripts/modules-check.sh is invoked. If it
> > happens again, the modpost will emit unclear error messages.
> >
> > To fix this issue completely, create *.mod in the same directory as
> > *.ko so that two threads never attempt to write to the same file.
> > $(MODVERDIR) is no longer needed.
> >
> > Since modules with directory paths are listed in modules.order, Kbuild
> > is still able to find *.mod files without additional descending.
> >
> > Signed-off-by: Masahiro Yamada <yamada.masahiro@socionext.com>
> > Acked-by: Nicolas Pitre <nico@fluxnic.net>
> >
>
> Hi Masahiro,
>
> I'm following this patchset changes as they will affect the klp-convert
> series [1] that the livepatching folks have been working on...
Empty files .tmp_versions/*.livepatch are touched
to keep track of 'LIVEPATCH_* := y', right?
Perhaps, adding a new field
to *.mod files might be cleaner.
> Just wondering if these other files should be checked for more MODVERDIR
> fallout:
>
> % grep -R 'tmp_versions'
> tools/power/cpupower/debug/kernel/Makefile: - rm -rf .tmp_versions* Module.symvers modules.order
> scripts/export_report.pl: while (<.tmp_versions/*.mod>) {
> scripts/adjust_autoksyms.sh:# .tmp_versions/*.mod files.
>
> export_report.pl is probably the only interesting one on this list.
Good catch. I will fix it.
> Also, can you cc me on subsequent patchset versions?
Yes, will do.
--
Best Regards
Masahiro Yamada
^ permalink raw reply
* [PATCH v5] Documentation/checkpatch: Prefer strscpy/strscpy_pad over strcpy/strlcpy/strncpy
From: NitinGote @ 2019-07-17 4:30 UTC (permalink / raw)
To: joe, corbet; +Cc: akpm, apw, keescook, linux-doc, kernel-hardening, Nitin Gote
From: Nitin Gote <nitin.r.gote@intel.com>
Added check in checkpatch.pl to
1. Deprecate strcpy() in favor of strscpy().
2. Deprecate strlcpy() in favor of strscpy().
3. Deprecate strncpy() in favor of strscpy() or strscpy_pad().
Updated strncpy() section in Documentation/process/deprecated.rst
to cover strscpy_pad() case.
Signed-off-by: Nitin Gote <nitin.r.gote@intel.com>
---
Documentation/process/deprecated.rst | 6 +++---
scripts/checkpatch.pl | 24 ++++++++++++++++++++++++
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/Documentation/process/deprecated.rst b/Documentation/process/deprecated.rst
index 49e0f64a3427..c348ef9d44f5 100644
--- a/Documentation/process/deprecated.rst
+++ b/Documentation/process/deprecated.rst
@@ -93,9 +93,9 @@ will be NUL terminated. This can lead to various linear read overflows
and other misbehavior due to the missing termination. It also NUL-pads the
destination buffer if the source contents are shorter than the destination
buffer size, which may be a needless performance penalty for callers using
-only NUL-terminated strings. The safe replacement is :c:func:`strscpy`.
-(Users of :c:func:`strscpy` still needing NUL-padding will need an
-explicit :c:func:`memset` added.)
+only NUL-terminated strings. In this case, the safe replacement is
+strscpy(). If, however, the destination buffer still needs NUL-padding,
+the safe replacement is strscpy_pad().
If a caller is using non-NUL-terminated strings, :c:func:`strncpy()` can
still be used, but destinations should be marked with the `__nonstring
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index bb28b178d929..1bb12127115d 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -605,6 +605,20 @@ foreach my $entry (keys %deprecated_apis) {
}
$deprecated_apis_search = "(?:${deprecated_apis_search})";
+our %deprecated_string_apis = (
+ "strcpy" => "strscpy",
+ "strlcpy" => "strscpy",
+ "strncpy" => "strscpy, strscpy_pad or for non-NUL-terminated strings, strncpy() can still be used, but destinations should be marked with __nonstring",
+);
+
+#Create a search pattern for all these strings apis to speed up a loop below
+our $deprecated_string_apis_search = "";
+foreach my $entry (keys %deprecated_string_apis) {
+ $deprecated_string_apis_search .= '|' if ($deprecated_string_apis_search ne "");
+ $deprecated_string_apis_search .= $entry;
+}
+$deprecated_string_apis_search = "(?:${deprecated_string_apis_search})";
+
our $mode_perms_world_writable = qr{
S_IWUGO |
S_IWOTH |
@@ -6446,6 +6460,16 @@ sub process {
"Deprecated use of '$deprecated_api', prefer '$new_api' instead\n" . $herecurr);
}
+# check for string deprecated apis
+ if ($line =~ /\b($deprecated_string_apis_search)\b\s*\(/) {
+ my $deprecated_string_api = $1;
+ my $new_api = $deprecated_string_apis{$deprecated_string_api};
+ my $msg_level = \&WARN;
+ $msg_level = \&CHK if ($file);
+ &{$msg_level}("DEPRECATED_API",
+ "Deprecated use of '$deprecated_string_api', prefer '$new_api' instead\n" . $herecurr);
+ }
+
# check for various structs that are normally const (ops, kgdb, device_tree)
# and avoid what seem like struct definitions 'struct foo {'
if ($line !~ /\bconst\b/ &&
--
2.17.1
^ permalink raw reply related
* [PATCH v2 2/2] f2fs: Support case-insensitive file name lookups
From: Daniel Rosenberg @ 2019-07-17 3:14 UTC (permalink / raw)
To: Jaegeuk Kim, Chao Yu, Jonathan Corbet, linux-f2fs-devel
Cc: linux-kernel, linux-doc, linux-fsdevel, kernel-team,
Daniel Rosenberg
In-Reply-To: <20190717031408.114104-1-drosen@google.com>
Modeled after commit b886ee3e778e ("ext4: Support case-insensitive file
name lookups")
"""
This patch implements the actual support for case-insensitive file name
lookups in f2fs, based on the feature bit and the encoding stored in the
superblock.
A filesystem that has the casefold feature set is able to configure
directories with the +F (F2FS_CASEFOLD_FL) attribute, enabling lookups
to succeed in that directory in a case-insensitive fashion, i.e: match
a directory entry even if the name used by userspace is not a byte per
byte match with the disk name, but is an equivalent case-insensitive
version of the Unicode string. This operation is called a
case-insensitive file name lookup.
The feature is configured as an inode attribute applied to directories
and inherited by its children. This attribute can only be enabled on
empty directories for filesystems that support the encoding feature,
thus preventing collision of file names that only differ by case.
* dcache handling:
For a +F directory, F2Fs only stores the first equivalent name dentry
used in the dcache. This is done to prevent unintentional duplication of
dentries in the dcache, while also allowing the VFS code to quickly find
the right entry in the cache despite which equivalent string was used in
a previous lookup, without having to resort to ->lookup().
d_hash() of casefolded directories is implemented as the hash of the
casefolded string, such that we always have a well-known bucket for all
the equivalencies of the same string. d_compare() uses the
utf8_strncasecmp() infrastructure, which handles the comparison of
equivalent, same case, names as well.
For now, negative lookups are not inserted in the dcache, since they
would need to be invalidated anyway, because we can't trust missing file
dentries. This is bad for performance but requires some leveraging of
the vfs layer to fix. We can live without that for now, and so does
everyone else.
* on-disk data:
Despite using a specific version of the name as the internal
representation within the dcache, the name stored and fetched from the
disk is a byte-per-byte match with what the user requested, making this
implementation 'name-preserving'. i.e. no actual information is lost
when writing to storage.
DX is supported by modifying the hashes used in +F directories to make
them case/encoding-aware. The new disk hashes are calculated as the
hash of the full casefolded string, instead of the string directly.
This allows us to efficiently search for file names in the htree without
requiring the user to provide an exact name.
* Dealing with invalid sequences:
By default, when a invalid UTF-8 sequence is identified, ext4 will treat
it as an opaque byte sequence, ignoring the encoding and reverting to
the old behavior for that unique file. This means that case-insensitive
file name lookup will not work only for that file. An optional bit can
be set in the superblock telling the filesystem code and userspace tools
to enforce the encoding. When that optional bit is set, any attempt to
create a file name using an invalid UTF-8 sequence will fail and return
an error to userspace.
* Normalization algorithm:
The UTF-8 algorithms used to compare strings in f2fs is implemented
in fs/unicode, and is based on a previous version developed by
SGI. It implements the Canonical decomposition (NFD) algorithm
described by the Unicode specification 12.1, or higher, combined with
the elimination of ignorable code points (NFDi) and full
case-folding (CF) as documented in fs/unicode/utf8_norm.c.
NFD seems to be the best normalization method for F2FS because:
- It has a lower cost than NFC/NFKC (which requires
decomposing to NFD as an intermediary step)
- It doesn't eliminate important semantic meaning like
compatibility decompositions.
Although:
- This implementation is not completely linguistic accurate, because
different languages have conflicting rules, which would require the
specialization of the filesystem to a given locale, which brings all
sorts of problems for removable media and for users who use more than
one language.
"""
Signed-off-by: Daniel Rosenberg <drosen@google.com>
---
fs/f2fs/dir.c | 133 ++++++++++++++++++++++++++++++++++++++++++-----
fs/f2fs/f2fs.h | 18 +++++--
fs/f2fs/file.c | 10 +++-
fs/f2fs/hash.c | 34 +++++++++++-
fs/f2fs/inline.c | 6 +--
fs/f2fs/inode.c | 4 +-
fs/f2fs/namei.c | 21 ++++++++
fs/f2fs/super.c | 5 ++
8 files changed, 208 insertions(+), 23 deletions(-)
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
index 85a1528f319f2..4d5eea2db1657 100644
--- a/fs/f2fs/dir.c
+++ b/fs/f2fs/dir.c
@@ -8,6 +8,7 @@
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/sched/signal.h>
+#include <linux/unicode.h>
#include "f2fs.h"
#include "node.h"
#include "acl.h"
@@ -81,7 +82,8 @@ static unsigned long dir_block_index(unsigned int level,
return bidx;
}
-static struct f2fs_dir_entry *find_in_block(struct page *dentry_page,
+static struct f2fs_dir_entry *find_in_block(struct inode *dir,
+ struct page *dentry_page,
struct fscrypt_name *fname,
f2fs_hash_t namehash,
int *max_slots,
@@ -94,20 +96,56 @@ static struct f2fs_dir_entry *find_in_block(struct page *dentry_page,
dentry_blk = (struct f2fs_dentry_block *)page_address(dentry_page);
make_dentry_ptr_block(NULL, &d, dentry_blk);
- de = f2fs_find_target_dentry(fname, namehash, max_slots, &d);
+ de = f2fs_find_target_dentry(dir, fname, namehash, max_slots, &d);
if (de)
*res_page = dentry_page;
return de;
}
-struct f2fs_dir_entry *f2fs_find_target_dentry(struct fscrypt_name *fname,
- f2fs_hash_t namehash, int *max_slots,
- struct f2fs_dentry_ptr *d)
+#ifdef CONFIG_UNICODE
+/*
+ * Test whether a case-insensitive directory entry matches the filename
+ * being searched for.
+ *
+ * Returns: 0 if the directory entry matches, more than 0 if it
+ * doesn't match or less than zero on error.
+ */
+int f2fs_ci_compare(const struct inode *parent, const struct qstr *name,
+ const struct qstr *entry)
+{
+ const struct f2fs_sb_info *sbi = F2FS_SB(parent->i_sb);
+ const struct unicode_map *um = sbi->s_encoding;
+ int ret;
+
+ ret = utf8_strncasecmp(um, name, entry);
+ if (ret < 0) {
+ /* Handle invalid character sequence as either an error
+ * or as an opaque byte sequence.
+ */
+ if (f2fs_has_strict_mode(sbi))
+ return -EINVAL;
+
+ if (name->len != entry->len)
+ return 1;
+
+ return !!memcmp(name->name, entry->name, name->len);
+ }
+
+ return ret;
+}
+#endif
+
+struct f2fs_dir_entry *f2fs_find_target_dentry(const struct inode *parent,
+ struct fscrypt_name *fname, f2fs_hash_t namehash,
+ int *max_slots, struct f2fs_dentry_ptr *d)
{
struct f2fs_dir_entry *de;
unsigned long bit_pos = 0;
int max_len = 0;
+#ifdef CONFIG_UNICODE
+ struct qstr entry;
+#endif
if (max_slots)
*max_slots = 0;
@@ -119,16 +157,29 @@ struct f2fs_dir_entry *f2fs_find_target_dentry(struct fscrypt_name *fname,
}
de = &d->dentry[bit_pos];
+#ifdef CONFIG_UNICODE
+ entry.name = d->filename[bit_pos];
+ entry.len = de->name_len;
+#endif
if (unlikely(!de->name_len)) {
bit_pos++;
continue;
}
+ if (de->hash_code == namehash) {
+#ifdef CONFIG_UNICODE
+ if (F2FS_SB(parent->i_sb)->s_encoding &&
+ IS_CASEFOLDED(parent) &&
+ !f2fs_ci_compare(parent,
+ fname->usr_fname, &entry))
+ goto found;
- if (de->hash_code == namehash &&
- fscrypt_match_name(fname, d->filename[bit_pos],
- le16_to_cpu(de->name_len)))
- goto found;
+#endif
+ if (de->hash_code == namehash &&
+ fscrypt_match_name(fname, d->filename[bit_pos],
+ le16_to_cpu(de->name_len)))
+ goto found;
+ }
if (max_slots && max_len > *max_slots)
*max_slots = max_len;
@@ -157,7 +208,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
struct f2fs_dir_entry *de = NULL;
bool room = false;
int max_slots;
- f2fs_hash_t namehash = f2fs_dentry_hash(&name, fname);
+ f2fs_hash_t namehash = f2fs_dentry_hash(dir, &name, fname);
nbucket = dir_buckets(level, F2FS_I(dir)->i_dir_level);
nblock = bucket_blocks(level);
@@ -179,8 +230,8 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
}
}
- de = find_in_block(dentry_page, fname, namehash, &max_slots,
- res_page);
+ de = find_in_block(dir, dentry_page, fname, namehash,
+ &max_slots, res_page);
if (de)
break;
@@ -246,10 +297,18 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
const struct qstr *child, struct page **res_page)
{
+ struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
struct f2fs_dir_entry *de = NULL;
struct fscrypt_name fname;
int err;
+#ifdef CONFIG_UNICODE
+ if (f2fs_has_strict_mode(sbi) && IS_CASEFOLDED(dir) &&
+ utf8_validate(sbi->s_encoding, child)) {
+ *res_page = ERR_PTR(-EINVAL);
+ return NULL;
+ }
+#endif
err = fscrypt_setup_filename(dir, child, 1, &fname);
if (err) {
if (err == -ENOENT)
@@ -504,7 +563,7 @@ int f2fs_add_regular_entry(struct inode *dir, const struct qstr *new_name,
level = 0;
slots = GET_DENTRY_SLOTS(new_name->len);
- dentry_hash = f2fs_dentry_hash(new_name, NULL);
+ dentry_hash = f2fs_dentry_hash(dir, new_name, NULL);
current_depth = F2FS_I(dir)->i_current_depth;
if (F2FS_I(dir)->chash == dentry_hash) {
@@ -943,3 +1002,51 @@ const struct file_operations f2fs_dir_operations = {
.compat_ioctl = f2fs_compat_ioctl,
#endif
};
+
+#ifdef CONFIG_UNICODE
+static int f2fs_d_compare(const struct dentry *dentry, unsigned int len,
+ const char *str, const struct qstr *name)
+{
+ struct qstr qstr = {.name = str, .len = len };
+
+ if (!IS_CASEFOLDED(dentry->d_parent->d_inode)) {
+ if (len != name->len)
+ return -1;
+ return memcmp(str, name, len);
+ }
+
+ return f2fs_ci_compare(dentry->d_parent->d_inode, name, &qstr);
+}
+
+static int f2fs_d_hash(const struct dentry *dentry, struct qstr *str)
+{
+ const struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb);
+ const struct unicode_map *um = sbi->s_encoding;
+ unsigned char *norm;
+ int len, ret = 0;
+
+ if (!IS_CASEFOLDED(dentry->d_inode))
+ return 0;
+
+ norm = kmalloc(PATH_MAX, GFP_ATOMIC);
+ if (!norm)
+ return -ENOMEM;
+
+ len = utf8_casefold(um, str, norm, PATH_MAX);
+ if (len < 0) {
+ if (f2fs_has_strict_mode(sbi))
+ ret = -EINVAL;
+ goto out;
+ }
+ str->hash = full_name_hash(dentry, norm, len);
+out:
+ kfree(norm);
+ return ret;
+}
+
+const struct dentry_operations f2fs_dentry_ops = {
+ .d_hash = f2fs_d_hash,
+ .d_compare = f2fs_d_compare,
+};
+#endif
+
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index c6c7904572d0d..500906108937c 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -2364,10 +2364,12 @@ static inline void f2fs_change_bit(unsigned int nr, char *addr)
#define F2FS_INDEX_FL 0x00001000 /* hash-indexed directory */
#define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */
#define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */
+#define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */
/* Flags that should be inherited by new inodes from their parent. */
#define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \
- F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL)
+ F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
+ F2FS_CASEFOLD_FL)
/* Flags that are appropriate for regular files (all but dir-specific ones). */
#define F2FS_REG_FLMASK (~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL))
@@ -2930,11 +2932,16 @@ int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
bool hot, bool set);
struct dentry *f2fs_get_parent(struct dentry *child);
+extern int f2fs_ci_compare(const struct inode *parent,
+ const struct qstr *name,
+ const struct qstr *entry);
+
/*
* dir.c
*/
unsigned char f2fs_get_de_type(struct f2fs_dir_entry *de);
-struct f2fs_dir_entry *f2fs_find_target_dentry(struct fscrypt_name *fname,
+struct f2fs_dir_entry *f2fs_find_target_dentry(const struct inode *parent,
+ struct fscrypt_name *fname,
f2fs_hash_t namehash, int *max_slots,
struct f2fs_dentry_ptr *d);
int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
@@ -2993,8 +3000,8 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
/*
* hash.c
*/
-f2fs_hash_t f2fs_dentry_hash(const struct qstr *name_info,
- struct fscrypt_name *fname);
+f2fs_hash_t f2fs_dentry_hash(const struct inode *dir,
+ const struct qstr *name_info, struct fscrypt_name *fname);
/*
* node.c
@@ -3437,6 +3444,9 @@ static inline void f2fs_destroy_root_stats(void) { }
#endif
extern const struct file_operations f2fs_dir_operations;
+#ifdef CONFIG_UNICODE
+extern const struct dentry_operations f2fs_dentry_ops;
+#endif
extern const struct file_operations f2fs_file_operations;
extern const struct inode_operations f2fs_file_inode_operations;
extern const struct address_space_operations f2fs_dblock_aops;
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index f8d46df8fa9ee..9bdef3aa38eab 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -1660,7 +1660,16 @@ static int f2fs_setflags_common(struct inode *inode, u32 iflags, u32 mask)
return -EPERM;
oldflags = fi->i_flags;
+ if ((iflags ^ oldflags) & F2FS_CASEFOLD_FL) {
+ if (!f2fs_sb_has_casefold(F2FS_I_SB(inode)))
+ return -EOPNOTSUPP;
+
+ if (!S_ISDIR(inode->i_mode))
+ return -ENOTDIR;
+ if (!f2fs_empty_dir(inode))
+ return -ENOTEMPTY;
+ }
if ((iflags ^ oldflags) & (F2FS_APPEND_FL | F2FS_IMMUTABLE_FL))
if (!capable(CAP_LINUX_IMMUTABLE))
return -EPERM;
@@ -1671,7 +1680,6 @@ static int f2fs_setflags_common(struct inode *inode, u32 iflags, u32 mask)
set_inode_flag(inode, FI_PROJ_INHERIT);
else
clear_inode_flag(inode, FI_PROJ_INHERIT);
-
inode->i_ctime = current_time(inode);
f2fs_set_inode_flags(inode);
f2fs_mark_inode_dirty_sync(inode, true);
diff --git a/fs/f2fs/hash.c b/fs/f2fs/hash.c
index cc82f142f811f..f5b8e02bde049 100644
--- a/fs/f2fs/hash.c
+++ b/fs/f2fs/hash.c
@@ -14,6 +14,7 @@
#include <linux/f2fs_fs.h>
#include <linux/cryptohash.h>
#include <linux/pagemap.h>
+#include <linux/unicode.h>
#include "f2fs.h"
@@ -67,7 +68,7 @@ static void str2hashbuf(const unsigned char *msg, size_t len,
*buf++ = pad;
}
-f2fs_hash_t f2fs_dentry_hash(const struct qstr *name_info,
+static f2fs_hash_t __f2fs_dentry_hash(const struct qstr *name_info,
struct fscrypt_name *fname)
{
__u32 hash;
@@ -103,3 +104,34 @@ f2fs_hash_t f2fs_dentry_hash(const struct qstr *name_info,
f2fs_hash = cpu_to_le32(hash & ~F2FS_HASH_COL_BIT);
return f2fs_hash;
}
+
+f2fs_hash_t f2fs_dentry_hash(const struct inode *dir,
+ const struct qstr *name_info, struct fscrypt_name *fname)
+{
+#ifdef CONFIG_UNICODE
+ const struct unicode_map *um = F2FS_SB(dir->i_sb)->s_encoding;
+ int r, dlen;
+ unsigned char *buff;
+ struct qstr *folded;
+
+ if (name_info->len && IS_CASEFOLDED(dir)) {
+ buff = kzalloc(sizeof(char) * PATH_MAX, GFP_KERNEL);
+ if (!buff)
+ return -ENOMEM;
+
+ dlen = utf8_casefold(um, name_info, buff, PATH_MAX);
+ if (dlen < 0) {
+ kfree(buff);
+ goto opaque_seq;
+ }
+ folded->name = buff;
+ folded->len = dlen;
+ r = __f2fs_dentry_hash(folded, fname);
+
+ kfree(buff);
+ return r;
+ }
+opaque_seq:
+#endif
+ return __f2fs_dentry_hash(name_info, fname);
+}
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index 3613efca8c00c..7cff67af4fadb 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -320,12 +320,12 @@ struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
return NULL;
}
- namehash = f2fs_dentry_hash(&name, fname);
+ namehash = f2fs_dentry_hash(dir, &name, fname);
inline_dentry = inline_data_addr(dir, ipage);
make_dentry_ptr_inline(dir, &d, inline_dentry);
- de = f2fs_find_target_dentry(fname, namehash, NULL, &d);
+ de = f2fs_find_target_dentry(dir, fname, namehash, NULL, &d);
unlock_page(ipage);
if (de)
*res_page = ipage;
@@ -580,7 +580,7 @@ int f2fs_add_inline_entry(struct inode *dir, const struct qstr *new_name,
f2fs_wait_on_page_writeback(ipage, NODE, true, true);
- name_hash = f2fs_dentry_hash(new_name, NULL);
+ name_hash = f2fs_dentry_hash(dir, new_name, NULL);
f2fs_update_dentry(ino, mode, &d, new_name, name_hash, bit_pos);
set_page_dirty(ipage);
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index a33d7a849b2df..9a1f0d6616577 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -46,9 +46,11 @@ void f2fs_set_inode_flags(struct inode *inode)
new_fl |= S_DIRSYNC;
if (file_is_encrypt(inode))
new_fl |= S_ENCRYPTED;
+ if (flags & F2FS_CASEFOLD_FL)
+ new_fl |= S_CASEFOLD;
inode_set_flags(inode, new_fl,
S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|
- S_ENCRYPTED);
+ S_ENCRYPTED|S_CASEFOLD);
}
static void __get_inode_rdev(struct inode *inode, struct f2fs_inode *ri)
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index c5b99042e6f2b..727de2f8620f2 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -489,6 +489,17 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
goto out_iput;
}
out_splice:
+#ifdef CONFIG_UNICODE
+ if (!inode && IS_CASEFOLDED(dir)) {
+ /* Eventually we want to call d_add_ci(dentry, NULL)
+ * for negative dentries in the encoding case as
+ * well. For now, prevent the negative dentry
+ * from being cached.
+ */
+ trace_f2fs_lookup_end(dir, dentry, ino, err);
+ return NULL;
+ }
+#endif
new = d_splice_alias(inode, dentry);
err = PTR_ERR_OR_ZERO(new);
trace_f2fs_lookup_end(dir, dentry, ino, err);
@@ -537,6 +548,16 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry)
goto fail;
}
f2fs_delete_entry(de, page, dir, inode);
+#ifdef CONFIG_UNICODE
+ /* VFS negative dentries are incompatible with Encoding and
+ * Case-insensitiveness. Eventually we'll want avoid
+ * invalidating the dentries here, alongside with returning the
+ * negative dentries at f2fs_lookup(), when it is better
+ * supported by the VFS for the CI case.
+ */
+ if (IS_CASEFOLDED(dir))
+ d_invalidate(dentry);
+#endif
f2fs_unlock_op(sbi);
if (IS_DIRSYNC(dir))
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 7927071ef5e95..edc8482a43604 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -3397,6 +3397,11 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
goto free_node_inode;
}
+#ifdef CONFIG_UNICODE
+ if (sbi->s_encoding)
+ sb->s_d_op = &f2fs_dentry_ops;
+#endif
+
sb->s_root = d_make_root(root); /* allocate root dentry */
if (!sb->s_root) {
err = -ENOMEM;
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v2 1/2] f2fs: include charset encoding information in the superblock
From: Daniel Rosenberg @ 2019-07-17 3:14 UTC (permalink / raw)
To: Jaegeuk Kim, Chao Yu, Jonathan Corbet, linux-f2fs-devel
Cc: linux-kernel, linux-doc, linux-fsdevel, kernel-team,
Daniel Rosenberg
In-Reply-To: <20190717031408.114104-1-drosen@google.com>
Add charset encoding to f2fs to support casefolding. It is modeled after
the same feature introduced in commit c83ad55eaa91 ("ext4: include charset
encoding information in the superblock")
Currently this is not compatible with encryption, similar to the current
ext4 imlpementation. This will change in the future.
From the ext4 patch:
"""
The s_encoding field stores a magic number indicating the encoding
format and version used globally by file and directory names in the
filesystem. The s_encoding_flags defines policies for using the charset
encoding, like how to handle invalid sequences. The magic number is
mapped to the exact charset table, but the mapping is specific to ext4.
Since we don't have any commitment to support old encodings, the only
encoding I am supporting right now is utf8-12.1.0.
The current implementation prevents the user from enabling encoding and
per-directory encryption on the same filesystem at the same time. The
incompatibility between these features lies in how we do efficient
directory searches when we cannot be sure the encryption of the user
provided fname will match the actual hash stored in the disk without
decrypting every directory entry, because of normalization cases. My
quickest solution is to simply block the concurrent use of these
features for now, and enable it later, once we have a better solution.
"""
Signed-off-by: Daniel Rosenberg <drosen@google.com>
---
fs/f2fs/f2fs.h | 6 +++
fs/f2fs/super.c | 81 +++++++++++++++++++++++++++++++++++++++++
include/linux/f2fs_fs.h | 9 ++++-
3 files changed, 95 insertions(+), 1 deletion(-)
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 17382da7f0bd9..c6c7904572d0d 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -153,6 +153,7 @@ struct f2fs_mount_info {
#define F2FS_FEATURE_LOST_FOUND 0x0200
#define F2FS_FEATURE_VERITY 0x0400 /* reserved */
#define F2FS_FEATURE_SB_CHKSUM 0x0800
+#define F2FS_FEATURE_CASEFOLD 0x1000
#define __F2FS_HAS_FEATURE(raw_super, mask) \
((raw_super->feature & cpu_to_le32(mask)) != 0)
@@ -1169,6 +1170,10 @@ struct f2fs_sb_info {
int valid_super_block; /* valid super block no */
unsigned long s_flag; /* flags for sbi */
struct mutex writepages; /* mutex for writepages() */
+#ifdef CONFIG_UNICODE
+ struct unicode_map *s_encoding;
+ __u16 s_encoding_flags;
+#endif
#ifdef CONFIG_BLK_DEV_ZONED
unsigned int blocks_per_blkz; /* F2FS blocks per zone */
@@ -3562,6 +3567,7 @@ F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO);
F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME);
F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND);
F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
+F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
#ifdef CONFIG_BLK_DEV_ZONED
static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 6de6cda440315..7927071ef5e95 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -23,6 +23,7 @@
#include <linux/f2fs_fs.h>
#include <linux/sysfs.h>
#include <linux/quota.h>
+#include <linux/unicode.h>
#include "f2fs.h"
#include "node.h"
@@ -222,6 +223,36 @@ void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...)
va_end(args);
}
+#ifdef CONFIG_UNICODE
+static const struct f2fs_sb_encodings {
+ __u16 magic;
+ char *name;
+ char *version;
+} f2fs_sb_encoding_map[] = {
+ {F2FS_ENC_UTF8_12_1, "utf8", "12.1.0"},
+};
+
+static int f2fs_sb_read_encoding(const struct f2fs_super_block *sb,
+ const struct f2fs_sb_encodings **encoding,
+ __u16 *flags)
+{
+ __u16 magic = le16_to_cpu(sb->s_encoding);
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(f2fs_sb_encoding_map); i++)
+ if (magic == f2fs_sb_encoding_map[i].magic)
+ break;
+
+ if (i >= ARRAY_SIZE(f2fs_sb_encoding_map))
+ return -EINVAL;
+
+ *encoding = &f2fs_sb_encoding_map[i];
+ *flags = le16_to_cpu(sb->s_encoding_flags);
+
+ return 0;
+}
+#endif
+
static inline void limit_reserve_root(struct f2fs_sb_info *sbi)
{
block_t limit = min((sbi->user_block_count << 1) / 1000,
@@ -798,6 +829,13 @@ static int parse_options(struct super_block *sb, char *options)
return -EINVAL;
}
#endif
+#ifndef CONFIG_UNICODE
+ if (f2fs_sb_has_casefold(sbi)) {
+ f2fs_err(sbi,
+ "Filesystem with casefold feature cannot be mounted without CONFIG_UNICODE");
+ return -EINVAL;
+ }
+#endif
if (F2FS_IO_SIZE_BITS(sbi) && !test_opt(sbi, LFS)) {
f2fs_err(sbi, "Should set mode=lfs with %uKB-sized IO",
@@ -1089,6 +1127,9 @@ static void f2fs_put_super(struct super_block *sb)
destroy_percpu_info(sbi);
for (i = 0; i < NR_PAGE_TYPE; i++)
kvfree(sbi->write_io[i]);
+#ifdef CONFIG_UNICODE
+ utf8_unload(sbi->s_encoding);
+#endif
kvfree(sbi);
}
@@ -3126,6 +3167,42 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
sb->s_maxbytes = sbi->max_file_blocks <<
le32_to_cpu(raw_super->log_blocksize);
sb->s_max_links = F2FS_LINK_MAX;
+#ifdef CONFIG_UNICODE
+ if (f2fs_sb_has_casefold(sbi) && !sbi->s_encoding) {
+ const struct f2fs_sb_encodings *encoding_info;
+ struct unicode_map *encoding;
+ __u16 encoding_flags;
+
+ if (f2fs_sb_has_encrypt(sbi)) {
+ f2fs_err(sbi,
+ "Can't mount with encoding and encryption");
+ goto free_options;
+ }
+
+ if (f2fs_sb_read_encoding(raw_super, &encoding_info,
+ &encoding_flags)) {
+ f2fs_err(sbi,
+ "Encoding requested by superblock is unknown");
+ goto free_options;
+ }
+
+ encoding = utf8_load(encoding_info->version);
+ if (IS_ERR(encoding)) {
+ f2fs_err(sbi,
+ "can't mount with superblock charset: %s-%s "
+ "not supported by the kernel. flags: 0x%x.",
+ encoding_info->name, encoding_info->version,
+ encoding_flags);
+ goto free_options;
+ }
+ f2fs_info(sbi, "Using encoding defined by superblock: "
+ "%s-%s with flags 0x%hx", encoding_info->name,
+ encoding_info->version?:"\b", encoding_flags);
+
+ sbi->s_encoding = encoding;
+ sbi->s_encoding_flags = encoding_flags;
+ }
+#endif
#ifdef CONFIG_QUOTA
sb->dq_op = &f2fs_quota_operations;
@@ -3477,6 +3554,10 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
free_bio_info:
for (i = 0; i < NR_PAGE_TYPE; i++)
kvfree(sbi->write_io[i]);
+
+#ifdef CONFIG_UNICODE
+ utf8_unload(sbi->s_encoding);
+#endif
free_options:
#ifdef CONFIG_QUOTA
for (i = 0; i < MAXQUOTAS; i++)
diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h
index 65559900d4d76..b7c9c7f721339 100644
--- a/include/linux/f2fs_fs.h
+++ b/include/linux/f2fs_fs.h
@@ -36,6 +36,11 @@
#define F2FS_MAX_QUOTAS 3
+#define F2FS_ENC_UTF8_12_1 1
+#define F2FS_ENC_STRICT_MODE_FL (1 << 0)
+#define f2fs_has_strict_mode(sbi) \
+ (sbi->s_encoding_flags & F2FS_ENC_STRICT_MODE_FL)
+
#define F2FS_IO_SIZE(sbi) (1 << F2FS_OPTION(sbi).write_io_size_bits) /* Blocks */
#define F2FS_IO_SIZE_KB(sbi) (1 << (F2FS_OPTION(sbi).write_io_size_bits + 2)) /* KB */
#define F2FS_IO_SIZE_BYTES(sbi) (1 << (F2FS_OPTION(sbi).write_io_size_bits + 12)) /* B */
@@ -109,7 +114,9 @@ struct f2fs_super_block {
struct f2fs_device devs[MAX_DEVICES]; /* device list */
__le32 qf_ino[F2FS_MAX_QUOTAS]; /* quota inode numbers */
__u8 hot_ext_count; /* # of hot file extension */
- __u8 reserved[310]; /* valid reserved region */
+ __le16 s_encoding; /* Filename charset encoding */
+ __le16 s_encoding_flags; /* Filename charset encoding flags */
+ __u8 reserved[306]; /* valid reserved region */
__le32 crc; /* checksum of superblock */
} __packed;
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v2 0/2] Casefolding in F2FS
From: Daniel Rosenberg @ 2019-07-17 3:14 UTC (permalink / raw)
To: Jaegeuk Kim, Chao Yu, Jonathan Corbet, linux-f2fs-devel
Cc: linux-kernel, linux-doc, linux-fsdevel, kernel-team,
Daniel Rosenberg
These patches are largely based on the casefolding patches for ext4
v2: Rebased patches again master, changed f2fs_msg to f2fs_info/f2fs_err
Daniel Rosenberg (2):
f2fs: include charset encoding information in the superblock
f2fs: Support case-insensitive file name lookups
fs/f2fs/dir.c | 133 ++++++++++++++++++++++++++++++++++++----
fs/f2fs/f2fs.h | 24 ++++++--
fs/f2fs/file.c | 10 ++-
fs/f2fs/hash.c | 34 +++++++++-
fs/f2fs/inline.c | 6 +-
fs/f2fs/inode.c | 4 +-
fs/f2fs/namei.c | 21 +++++++
fs/f2fs/super.c | 86 ++++++++++++++++++++++++++
include/linux/f2fs_fs.h | 9 ++-
9 files changed, 303 insertions(+), 24 deletions(-)
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply
* [PATCH v11 06/18] kbuild: enable building KUnit
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Brendan Higgins, Michal Marek
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
KUnit is a new unit testing framework for the kernel and when used is
built into the kernel as a part of it. Add KUnit to the root Kconfig and
Makefile to allow it to be actually built.
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Acked-by: Masahiro Yamada <yamada.masahiro@socionext.com>
Cc: Michal Marek <michal.lkml@markovi.net>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
Reviewed-by: Stephen Boyd <sboyd@kernel.org>
---
Kconfig | 2 ++
Makefile | 2 ++
2 files changed, 4 insertions(+)
diff --git a/Kconfig b/Kconfig
index 48a80beab6853..10428501edb78 100644
--- a/Kconfig
+++ b/Kconfig
@@ -30,3 +30,5 @@ source "crypto/Kconfig"
source "lib/Kconfig"
source "lib/Kconfig.debug"
+
+source "kunit/Kconfig"
diff --git a/Makefile b/Makefile
index 3e4868a6498b2..0ce1a8a2b6fec 100644
--- a/Makefile
+++ b/Makefile
@@ -993,6 +993,8 @@ PHONY += prepare0
ifeq ($(KBUILD_EXTMOD),)
core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/
+core-$(CONFIG_KUNIT) += kunit/
+
vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \
$(core-y) $(core-m) $(drivers-y) $(drivers-m) \
$(net-y) $(net-m) $(libs-y) $(libs-m) $(virt-y)))
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 09/18] kunit: test: add support for test abort
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Brendan Higgins
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
Add support for aborting/bailing out of test cases, which is needed for
implementing assertions.
An assertion is like an expectation, but bails out of the test case
early if the assertion is not met. The idea with assertions is that you
use them to state all the preconditions for your test. Logically
speaking, these are the premises of the test case, so if a premise isn't
true, there is no point in continuing the test case because there are no
conclusions that can be drawn without the premises. Whereas, the
expectation is the thing you are trying to prove.
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
---
include/kunit/test.h | 16 ++++
include/kunit/try-catch.h | 69 +++++++++++++++
kunit/Makefile | 3 +-
kunit/test.c | 176 ++++++++++++++++++++++++++++++++++----
kunit/try-catch.c | 95 ++++++++++++++++++++
5 files changed, 343 insertions(+), 16 deletions(-)
create mode 100644 include/kunit/try-catch.h
create mode 100644 kunit/try-catch.c
diff --git a/include/kunit/test.h b/include/kunit/test.h
index 6ae585478730a..1daa4510bb960 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -13,6 +13,7 @@
#include <linux/types.h>
#include <linux/slab.h>
#include <kunit/kunit-stream.h>
+#include <kunit/try-catch.h>
struct kunit_resource;
@@ -167,6 +168,7 @@ struct kunit {
/* private: internal use only. */
const char *name; /* Read only after initialization! */
+ struct kunit_try_catch try_catch;
/*
* success starts as true, and may only be set to false during a test
* case; thus, it is safe to update this across multiple threads using
@@ -176,6 +178,11 @@ struct kunit {
*/
bool success; /* Read only after test_case finishes! */
spinlock_t lock; /* Gaurds all mutable test state. */
+ /*
+ * death_test may be both set and unset from multiple threads in a test
+ * case.
+ */
+ bool death_test; /* Protected by lock. */
/*
* Because resources is a list that may be updated multiple times (with
* new resources) from any thread associated with a test case, we must
@@ -184,10 +191,19 @@ struct kunit {
struct list_head resources; /* Protected by lock. */
};
+static inline void kunit_set_death_test(struct kunit *test, bool death_test)
+{
+ spin_lock(&test->lock);
+ test->death_test = death_test;
+ spin_unlock(&test->lock);
+}
+
void kunit_init_test(struct kunit *test, const char *name);
void kunit_fail(struct kunit *test, struct kunit_stream *stream);
+void kunit_abort(struct kunit *test);
+
int kunit_run_tests(struct kunit_suite *suite);
/**
diff --git a/include/kunit/try-catch.h b/include/kunit/try-catch.h
new file mode 100644
index 0000000000000..8a414a9af0b64
--- /dev/null
+++ b/include/kunit/try-catch.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * An API to allow a function, that may fail, to be executed, and recover in a
+ * controlled manner.
+ *
+ * Copyright (C) 2019, Google LLC.
+ * Author: Brendan Higgins <brendanhiggins@google.com>
+ */
+
+#ifndef _KUNIT_TRY_CATCH_H
+#define _KUNIT_TRY_CATCH_H
+
+#include <linux/types.h>
+
+typedef void (*kunit_try_catch_func_t)(void *);
+
+struct kunit;
+
+/*
+ * struct kunit_try_catch - provides a generic way to run code which might fail.
+ * @context: used to pass user data to the try and catch functions.
+ *
+ * kunit_try_catch provides a generic, architecture independent way to execute
+ * an arbitrary function of type kunit_try_catch_func_t which may bail out by
+ * calling kunit_try_catch_throw(). If kunit_try_catch_throw() is called, @try
+ * is stopped at the site of invocation and @catch is catch is called.
+ *
+ * struct kunit_try_catch provides a generic interface for the functionality
+ * needed to implement kunit->abort() which in turn is needed for implementing
+ * assertions. Assertions allow stating a precondition for a test simplifying
+ * how test cases are written and presented.
+ *
+ * Assertions are like expectations, except they abort (call
+ * kunit_try_catch_throw()) when the specified condition is not met. This is
+ * useful when you look at a test case as a logical statement about some piece
+ * of code, where assertions are the premises for the test case, and the
+ * conclusion is a set of predicates, rather expectations, that must all be
+ * true. If your premises are violated, it does not makes sense to continue.
+ */
+struct kunit_try_catch {
+ /* private: internal use only. */
+ struct kunit *test;
+ struct completion *try_completion;
+ int try_result;
+ kunit_try_catch_func_t try;
+ kunit_try_catch_func_t catch;
+ void *context;
+};
+
+void kunit_try_catch_init(struct kunit_try_catch *try_catch,
+ struct kunit *test,
+ kunit_try_catch_func_t try,
+ kunit_try_catch_func_t catch);
+
+void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context);
+
+void __noreturn kunit_try_catch_throw(struct kunit_try_catch *try_catch);
+
+static inline int kunit_try_catch_get_result(struct kunit_try_catch *try_catch)
+{
+ return try_catch->try_result;
+}
+
+/*
+ * Exposed for testing only.
+ */
+void kunit_generic_try_catch_init(struct kunit_try_catch *try_catch);
+
+#endif /* _KUNIT_TRY_CATCH_H */
diff --git a/kunit/Makefile b/kunit/Makefile
index 60a9ea6cb4697..1f7680cfa11ad 100644
--- a/kunit/Makefile
+++ b/kunit/Makefile
@@ -1,6 +1,7 @@
obj-$(CONFIG_KUNIT) += test.o \
string-stream.o \
- kunit-stream.o
+ kunit-stream.o \
+ try-catch.o
obj-$(CONFIG_KUNIT_TEST) += string-stream-test.o
diff --git a/kunit/test.c b/kunit/test.c
index 1f94a9224b03e..12db50b221781 100644
--- a/kunit/test.c
+++ b/kunit/test.c
@@ -7,13 +7,26 @@
*/
#include <linux/kernel.h>
+#include <linux/sched/debug.h>
#include <kunit/test.h>
+#include <kunit/try-catch.h>
static void kunit_set_failure(struct kunit *test)
{
WRITE_ONCE(test->success, false);
}
+static bool kunit_get_death_test(struct kunit *test)
+{
+ bool death_test;
+
+ spin_lock(&test->lock);
+ death_test = test->death_test;
+ spin_unlock(&test->lock);
+
+ return death_test;
+}
+
static int kunit_vprintk_emit(int level, const char *fmt, va_list args)
{
return vprintk_emit(0, level, NULL, 0, fmt, args);
@@ -126,42 +139,175 @@ void kunit_fail(struct kunit *test, struct kunit_stream *stream)
kunit_stream_commit(stream);
}
+void __noreturn kunit_abort(struct kunit *test)
+{
+ kunit_set_death_test(test, true);
+
+ kunit_try_catch_throw(&test->try_catch);
+
+ /*
+ * Throw could not abort from test.
+ *
+ * XXX: we should never reach this line! As kunit_try_catch_throw is
+ * marked __noreturn.
+ */
+ WARN_ONCE(true, "Throw could not abort from test!\n");
+}
+
void kunit_init_test(struct kunit *test, const char *name)
{
spin_lock_init(&test->lock);
INIT_LIST_HEAD(&test->resources);
test->name = name;
test->success = true;
+ test->death_test = false;
}
/*
- * Performs all logic to run a test case.
+ * Initializes and runs test case. Does not clean up or do post validations.
*/
-static void kunit_run_case(struct kunit_suite *suite,
- struct kunit_case *test_case)
+static void kunit_run_case_internal(struct kunit *test,
+ struct kunit_suite *suite,
+ struct kunit_case *test_case)
{
- struct kunit test;
-
- kunit_init_test(&test, test_case->name);
-
if (suite->init) {
int ret;
- ret = suite->init(&test);
+ ret = suite->init(test);
if (ret) {
- kunit_err(&test, "failed to initialize: %d\n", ret);
- kunit_set_failure(&test);
- test_case->success = test.success;
+ kunit_err(test, "failed to initialize: %d\n", ret);
+ kunit_set_failure(test);
return;
}
}
- test_case->run_case(&test);
+ test_case->run_case(test);
+}
+
+static void kunit_case_internal_cleanup(struct kunit *test)
+{
+ kunit_cleanup(test);
+}
+/*
+ * Performs post validations and cleanup after a test case was run.
+ * XXX: Should ONLY BE CALLED AFTER kunit_run_case_internal!
+ */
+static void kunit_run_case_cleanup(struct kunit *test,
+ struct kunit_suite *suite)
+{
if (suite->exit)
- suite->exit(&test);
+ suite->exit(test);
+
+ kunit_case_internal_cleanup(test);
+}
+
+/*
+ * Handles an unexpected crash in a test case.
+ */
+static void kunit_handle_test_crash(struct kunit *test,
+ struct kunit_suite *suite,
+ struct kunit_case *test_case)
+{
+ kunit_err(test, "kunit test case crashed!");
+ /*
+ * TODO(brendanhiggins@google.com): This prints the stack trace up
+ * through this frame, not up to the frame that caused the crash.
+ */
+ show_stack(NULL, NULL);
+
+ kunit_case_internal_cleanup(test);
+}
+
+struct kunit_try_catch_context {
+ struct kunit *test;
+ struct kunit_suite *suite;
+ struct kunit_case *test_case;
+};
+
+static void kunit_try_run_case(void *data)
+{
+ struct kunit_try_catch_context *ctx = data;
+ struct kunit *test = ctx->test;
+ struct kunit_suite *suite = ctx->suite;
+ struct kunit_case *test_case = ctx->test_case;
+
+ /*
+ * kunit_run_case_internal may encounter a fatal error; if it does,
+ * abort will be called, this thread will exit, and finally the parent
+ * thread will resume control and handle any necessary clean up.
+ */
+ kunit_run_case_internal(test, suite, test_case);
+ /* This line may never be reached. */
+ kunit_run_case_cleanup(test, suite);
+}
+
+static void kunit_catch_run_case(void *data)
+{
+ struct kunit_try_catch_context *ctx = data;
+ struct kunit *test = ctx->test;
+ struct kunit_suite *suite = ctx->suite;
+ struct kunit_case *test_case = ctx->test_case;
+ int try_exit_code = kunit_try_catch_get_result(&test->try_catch);
+
+ if (try_exit_code) {
+ kunit_set_failure(test);
+ /*
+ * Test case could not finish, we have no idea what state it is
+ * in, so don't do clean up.
+ */
+ if (try_exit_code == -ETIMEDOUT)
+ kunit_err(test, "test case timed out\n");
+ /*
+ * Unknown internal error occurred preventing test case from
+ * running, so there is nothing to clean up.
+ */
+ else
+ kunit_err(test, "internal error occurred preventing test case from running: %d\n",
+ try_exit_code);
+ return;
+ }
+
+ if (kunit_get_death_test(test)) {
+ /*
+ * EXPECTED DEATH: kunit_run_case_internal encountered
+ * anticipated fatal error. Everything should be in a safe
+ * state.
+ */
+ kunit_run_case_cleanup(test, suite);
+ } else {
+ /*
+ * UNEXPECTED DEATH: kunit_run_case_internal encountered an
+ * unanticipated fatal error. We have no idea what the state of
+ * the test case is in.
+ */
+ kunit_handle_test_crash(test, suite, test_case);
+ kunit_set_failure(test);
+ }
+}
+
+/*
+ * Performs all logic to run a test case. It also catches most errors that
+ * occurs in a test case and reports them as failures.
+ */
+static void kunit_run_case_catch_errors(struct kunit_suite *suite,
+ struct kunit_case *test_case)
+{
+ struct kunit_try_catch_context context;
+ struct kunit_try_catch *try_catch;
+ struct kunit test;
+
+ kunit_init_test(&test, test_case->name);
+ try_catch = &test.try_catch;
- kunit_cleanup(&test);
+ kunit_try_catch_init(try_catch,
+ &test,
+ kunit_try_run_case,
+ kunit_catch_run_case);
+ context.test = &test;
+ context.suite = suite;
+ context.test_case = test_case;
+ kunit_try_catch_run(try_catch, &context);
test_case->success = test.success;
}
@@ -174,7 +320,7 @@ int kunit_run_tests(struct kunit_suite *suite)
kunit_print_subtest_start(suite);
for (test_case = suite->test_cases; test_case->run_case; test_case++) {
- kunit_run_case(suite, test_case);
+ kunit_run_case_catch_errors(suite, test_case);
kunit_print_test_case_ok_not_ok(test_case, test_case_count++);
}
diff --git a/kunit/try-catch.c b/kunit/try-catch.c
new file mode 100644
index 0000000000000..de580f074387b
--- /dev/null
+++ b/kunit/try-catch.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * An API to allow a function, that may fail, to be executed, and recover in a
+ * controlled manner.
+ *
+ * Copyright (C) 2019, Google LLC.
+ * Author: Brendan Higgins <brendanhiggins@google.com>
+ */
+
+#include <kunit/try-catch.h>
+#include <kunit/test.h>
+#include <linux/completion.h>
+#include <linux/kthread.h>
+
+void __noreturn kunit_try_catch_throw(struct kunit_try_catch *try_catch)
+{
+ try_catch->try_result = -EFAULT;
+ complete_and_exit(try_catch->try_completion, -EFAULT);
+}
+
+static int kunit_generic_run_threadfn_adapter(void *data)
+{
+ struct kunit_try_catch *try_catch = data;
+
+ try_catch->try(try_catch->context);
+
+ complete_and_exit(try_catch->try_completion, 0);
+}
+
+void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context)
+{
+ DECLARE_COMPLETION_ONSTACK(try_completion);
+ struct kunit *test = try_catch->test;
+ struct task_struct *task_struct;
+ int exit_code, status;
+
+ try_catch->context = context;
+ try_catch->try_completion = &try_completion;
+ try_catch->try_result = 0;
+ task_struct = kthread_run(kunit_generic_run_threadfn_adapter,
+ try_catch,
+ "kunit_try_catch_thread");
+ if (IS_ERR(task_struct)) {
+ try_catch->catch(try_catch->context);
+ return;
+ }
+
+ /*
+ * TODO(brendanhiggins@google.com): We should probably have some type of
+ * variable timeout here. The only question is what that timeout value
+ * should be.
+ *
+ * The intention has always been, at some point, to be able to label
+ * tests with some type of size bucket (unit/small, integration/medium,
+ * large/system/end-to-end, etc), where each size bucket would get a
+ * default timeout value kind of like what Bazel does:
+ * https://docs.bazel.build/versions/master/be/common-definitions.html#test.size
+ * There is still some debate to be had on exactly how we do this. (For
+ * one, we probably want to have some sort of test runner level
+ * timeout.)
+ *
+ * For more background on this topic, see:
+ * https://mike-bland.com/2011/11/01/small-medium-large.html
+ */
+ status = wait_for_completion_timeout(&try_completion,
+ 300 * MSEC_PER_SEC); /* 5 min */
+ if (status < 0) {
+ kunit_err(test, "try timed out\n");
+ try_catch->try_result = -ETIMEDOUT;
+ }
+
+ exit_code = try_catch->try_result;
+
+ if (!exit_code)
+ return;
+
+ if (exit_code == -EFAULT)
+ try_catch->try_result = 0;
+ else if (exit_code == -EINTR)
+ kunit_err(test, "wake_up_process() was never called\n");
+ else if (exit_code)
+ kunit_err(test, "Unknown error: %d\n", exit_code);
+
+ try_catch->catch(try_catch->context);
+}
+
+void kunit_try_catch_init(struct kunit_try_catch *try_catch,
+ struct kunit *test,
+ kunit_try_catch_func_t try,
+ kunit_try_catch_func_t catch)
+{
+ try_catch->test = test;
+ try_catch->try = try;
+ try_catch->catch = catch;
+}
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 12/18] kunit: test: add tests for KUnit managed resources
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Avinash Kondareddy, Brendan Higgins
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
From: Avinash Kondareddy <akndr41@gmail.com>
Add unit tests for KUnit managed resources. KUnit managed resources
(struct kunit_resource) are resources that are automatically cleaned up
at the end of a KUnit test, similar to the concept of devm_* managed
resources.
Signed-off-by: Avinash Kondareddy <akndr41@gmail.com>
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
---
kunit/test-test.c | 225 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 225 insertions(+)
diff --git a/kunit/test-test.c b/kunit/test-test.c
index 058f3fb37458a..725f1486376fa 100644
--- a/kunit/test-test.c
+++ b/kunit/test-test.c
@@ -101,3 +101,228 @@ static struct kunit_suite kunit_try_catch_test_suite = {
.test_cases = kunit_try_catch_test_cases,
};
kunit_test_suite(kunit_try_catch_test_suite);
+
+/*
+ * Context for testing test managed resources
+ * is_resource_initialized is used to test arbitrary resources
+ */
+struct kunit_test_resource_context {
+ struct kunit test;
+ bool is_resource_initialized;
+ int allocate_order[2];
+ int free_order[2];
+};
+
+static int fake_resource_init(struct kunit_resource *res, void *context)
+{
+ struct kunit_test_resource_context *ctx = context;
+
+ res->allocation = &ctx->is_resource_initialized;
+ ctx->is_resource_initialized = true;
+ return 0;
+}
+
+static void fake_resource_free(struct kunit_resource *res)
+{
+ bool *is_resource_initialized = res->allocation;
+
+ *is_resource_initialized = false;
+}
+
+static void kunit_resource_test_init_resources(struct kunit *test)
+{
+ struct kunit_test_resource_context *ctx = test->priv;
+
+ kunit_init_test(&ctx->test, "testing_test_init_test");
+
+ KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources));
+}
+
+static void kunit_resource_test_alloc_resource(struct kunit *test)
+{
+ struct kunit_test_resource_context *ctx = test->priv;
+ struct kunit_resource *res;
+ kunit_resource_free_t free = fake_resource_free;
+
+ res = kunit_alloc_and_get_resource(&ctx->test,
+ fake_resource_init,
+ fake_resource_free,
+ GFP_KERNEL,
+ ctx);
+
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res);
+ KUNIT_EXPECT_PTR_EQ(test,
+ &ctx->is_resource_initialized,
+ (bool *) res->allocation);
+ KUNIT_EXPECT_TRUE(test, list_is_last(&res->node, &ctx->test.resources));
+ KUNIT_EXPECT_PTR_EQ(test, free, res->free);
+}
+
+static void kunit_resource_test_free_resource(struct kunit *test)
+{
+ struct kunit_test_resource_context *ctx = test->priv;
+ struct kunit_resource *res = kunit_alloc_and_get_resource(
+ &ctx->test,
+ fake_resource_init,
+ fake_resource_free,
+ GFP_KERNEL,
+ ctx);
+
+ kunit_free_resource(&ctx->test, res);
+
+ KUNIT_EXPECT_FALSE(test, ctx->is_resource_initialized);
+ KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources));
+}
+
+static void kunit_resource_test_cleanup_resources(struct kunit *test)
+{
+ int i;
+ struct kunit_test_resource_context *ctx = test->priv;
+ struct kunit_resource *resources[5];
+
+ for (i = 0; i < ARRAY_SIZE(resources); i++) {
+ resources[i] = kunit_alloc_and_get_resource(&ctx->test,
+ fake_resource_init,
+ fake_resource_free,
+ GFP_KERNEL,
+ ctx);
+ }
+
+ kunit_cleanup(&ctx->test);
+
+ KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources));
+}
+
+static void kunit_resource_test_mark_order(int order_array[],
+ size_t order_size,
+ int key)
+{
+ int i;
+
+ for (i = 0; i < order_size && order_array[i]; i++)
+ ;
+
+ order_array[i] = key;
+}
+
+#define KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, order_field, key) \
+ kunit_resource_test_mark_order(ctx->order_field, \
+ ARRAY_SIZE(ctx->order_field), \
+ key)
+
+static int fake_resource_2_init(struct kunit_resource *res, void *context)
+{
+ struct kunit_test_resource_context *ctx = context;
+
+ KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 2);
+
+ res->allocation = ctx;
+
+ return 0;
+}
+
+static void fake_resource_2_free(struct kunit_resource *res)
+{
+ struct kunit_test_resource_context *ctx = res->allocation;
+
+ KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 2);
+}
+
+static int fake_resource_1_init(struct kunit_resource *res, void *context)
+{
+ struct kunit_test_resource_context *ctx = context;
+
+ kunit_alloc_and_get_resource(&ctx->test,
+ fake_resource_2_init,
+ fake_resource_2_free,
+ GFP_KERNEL,
+ ctx);
+
+ KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 1);
+
+ res->allocation = ctx;
+
+ return 0;
+}
+
+static void fake_resource_1_free(struct kunit_resource *res)
+{
+ struct kunit_test_resource_context *ctx = res->allocation;
+
+ KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 1);
+}
+
+/*
+ * TODO(brendanhiggins@google.com): replace the arrays that keep track of the
+ * order of allocation and freeing with strict mocks using the IN_SEQUENCE macro
+ * to assert allocation and freeing order when the feature becomes available.
+ */
+static void kunit_resource_test_proper_free_ordering(struct kunit *test)
+{
+ struct kunit_test_resource_context *ctx = test->priv;
+
+ /* fake_resource_1 allocates a fake_resource_2 in its init. */
+ kunit_alloc_and_get_resource(&ctx->test,
+ fake_resource_1_init,
+ fake_resource_1_free,
+ GFP_KERNEL,
+ ctx);
+
+ /*
+ * Since fake_resource_2_init calls KUNIT_RESOURCE_TEST_MARK_ORDER
+ * before returning to fake_resource_1_init, it should be the first to
+ * put its key in the allocate_order array.
+ */
+ KUNIT_EXPECT_EQ(test, ctx->allocate_order[0], 2);
+ KUNIT_EXPECT_EQ(test, ctx->allocate_order[1], 1);
+
+ kunit_cleanup(&ctx->test);
+
+ /*
+ * Because fake_resource_2 finishes allocation before fake_resource_1,
+ * fake_resource_1 should be freed first since it could depend on
+ * fake_resource_2.
+ */
+ KUNIT_EXPECT_EQ(test, ctx->free_order[0], 1);
+ KUNIT_EXPECT_EQ(test, ctx->free_order[1], 2);
+}
+
+static int kunit_resource_test_init(struct kunit *test)
+{
+ struct kunit_test_resource_context *ctx =
+ kzalloc(sizeof(*ctx), GFP_KERNEL);
+
+ if (!ctx)
+ return -ENOMEM;
+
+ test->priv = ctx;
+
+ kunit_init_test(&ctx->test, "test_test_context");
+
+ return 0;
+}
+
+static void kunit_resource_test_exit(struct kunit *test)
+{
+ struct kunit_test_resource_context *ctx = test->priv;
+
+ kunit_cleanup(&ctx->test);
+ kfree(ctx);
+}
+
+static struct kunit_case kunit_resource_test_cases[] = {
+ KUNIT_CASE(kunit_resource_test_init_resources),
+ KUNIT_CASE(kunit_resource_test_alloc_resource),
+ KUNIT_CASE(kunit_resource_test_free_resource),
+ KUNIT_CASE(kunit_resource_test_cleanup_resources),
+ KUNIT_CASE(kunit_resource_test_proper_free_ordering),
+ {}
+};
+
+static struct kunit_suite kunit_resource_test_suite = {
+ .name = "kunit-resource-test",
+ .init = kunit_resource_test_init,
+ .exit = kunit_resource_test_exit,
+ .test_cases = kunit_resource_test_cases,
+};
+kunit_test_suite(kunit_resource_test_suite);
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 15/18] Documentation: kunit: add documentation for KUnit
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Brendan Higgins, Felix Guo, Jonathan Corbet
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
Add documentation for KUnit, the Linux kernel unit testing framework.
- Add intro and usage guide for KUnit
- Add API reference
Signed-off-by: Felix Guo <felixguoxiuping@gmail.com>
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
---
Documentation/dev-tools/index.rst | 1 +
Documentation/dev-tools/kunit/api/index.rst | 16 +
Documentation/dev-tools/kunit/api/test.rst | 14 +
Documentation/dev-tools/kunit/faq.rst | 62 +++
Documentation/dev-tools/kunit/index.rst | 79 +++
Documentation/dev-tools/kunit/start.rst | 180 ++++++
Documentation/dev-tools/kunit/usage.rst | 575 ++++++++++++++++++++
7 files changed, 927 insertions(+)
create mode 100644 Documentation/dev-tools/kunit/api/index.rst
create mode 100644 Documentation/dev-tools/kunit/api/test.rst
create mode 100644 Documentation/dev-tools/kunit/faq.rst
create mode 100644 Documentation/dev-tools/kunit/index.rst
create mode 100644 Documentation/dev-tools/kunit/start.rst
create mode 100644 Documentation/dev-tools/kunit/usage.rst
diff --git a/Documentation/dev-tools/index.rst b/Documentation/dev-tools/index.rst
index b0522a4dd1073..09dee10d25928 100644
--- a/Documentation/dev-tools/index.rst
+++ b/Documentation/dev-tools/index.rst
@@ -24,6 +24,7 @@ whole; patches welcome!
gdb-kernel-debugging
kgdb
kselftest
+ kunit/index
.. only:: subproject and html
diff --git a/Documentation/dev-tools/kunit/api/index.rst b/Documentation/dev-tools/kunit/api/index.rst
new file mode 100644
index 0000000000000..9b9bffe5d41a0
--- /dev/null
+++ b/Documentation/dev-tools/kunit/api/index.rst
@@ -0,0 +1,16 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=============
+API Reference
+=============
+.. toctree::
+
+ test
+
+This section documents the KUnit kernel testing API. It is divided into the
+following sections:
+
+================================= ==============================================
+:doc:`test` documents all of the standard testing API
+ excluding mocking or mocking related features.
+================================= ==============================================
diff --git a/Documentation/dev-tools/kunit/api/test.rst b/Documentation/dev-tools/kunit/api/test.rst
new file mode 100644
index 0000000000000..d0ce19b1e1185
--- /dev/null
+++ b/Documentation/dev-tools/kunit/api/test.rst
@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+========
+Test API
+========
+
+This file documents all of the standard testing API excluding mocking or mocking
+related features.
+
+.. kernel-doc:: include/kunit/test.h
+ :internal:
+
+.. kernel-doc:: include/kunit/kunit-stream.h
+ :internal:
diff --git a/Documentation/dev-tools/kunit/faq.rst b/Documentation/dev-tools/kunit/faq.rst
new file mode 100644
index 0000000000000..bf2095112d899
--- /dev/null
+++ b/Documentation/dev-tools/kunit/faq.rst
@@ -0,0 +1,62 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==========================
+Frequently Asked Questions
+==========================
+
+How is this different from Autotest, kselftest, etc?
+====================================================
+KUnit is a unit testing framework. Autotest, kselftest (and some others) are
+not.
+
+A `unit test <https://martinfowler.com/bliki/UnitTest.html>`_ is supposed to
+test a single unit of code in isolation, hence the name. A unit test should be
+the finest granularity of testing and as such should allow all possible code
+paths to be tested in the code under test; this is only possible if the code
+under test is very small and does not have any external dependencies outside of
+the test's control like hardware.
+
+There are no testing frameworks currently available for the kernel that do not
+require installing the kernel on a test machine or in a VM and all require
+tests to be written in userspace and run on the kernel under test; this is true
+for Autotest, kselftest, and some others, disqualifying any of them from being
+considered unit testing frameworks.
+
+Does KUnit support running on architectures other than UML?
+===========================================================
+
+Yes, well, mostly.
+
+For the most part, the KUnit core framework (what you use to write the tests)
+can compile to any architecture; it compiles like just another part of the
+kernel and runs when the kernel boots. However, there is some infrastructure,
+like the KUnit Wrapper (``tools/testing/kunit/kunit.py``) that does not support
+other architectures.
+
+In short, this means that, yes, you can run KUnit on other architectures, but
+it might require more work than using KUnit on UML.
+
+For more information, see :ref:`kunit-on-non-uml`.
+
+What is the difference between a unit test and these other kinds of tests?
+==========================================================================
+Most existing tests for the Linux kernel would be categorized as an integration
+test, or an end-to-end test.
+
+- A unit test is supposed to test a single unit of code in isolation, hence the
+ name. A unit test should be the finest granularity of testing and as such
+ should allow all possible code paths to be tested in the code under test; this
+ is only possible if the code under test is very small and does not have any
+ external dependencies outside of the test's control like hardware.
+- An integration test tests the interaction between a minimal set of components,
+ usually just two or three. For example, someone might write an integration
+ test to test the interaction between a driver and a piece of hardware, or to
+ test the interaction between the userspace libraries the kernel provides and
+ the kernel itself; however, one of these tests would probably not test the
+ entire kernel along with hardware interactions and interactions with the
+ userspace.
+- An end-to-end test usually tests the entire system from the perspective of the
+ code under test. For example, someone might write an end-to-end test for the
+ kernel by installing a production configuration of the kernel on production
+ hardware with a production userspace and then trying to exercise some behavior
+ that depends on interactions between the hardware, the kernel, and userspace.
diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst
new file mode 100644
index 0000000000000..a317ab45bfe2d
--- /dev/null
+++ b/Documentation/dev-tools/kunit/index.rst
@@ -0,0 +1,79 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=========================================
+KUnit - Unit Testing for the Linux Kernel
+=========================================
+
+.. toctree::
+ :maxdepth: 2
+
+ start
+ usage
+ api/index
+ faq
+
+What is KUnit?
+==============
+
+KUnit is a lightweight unit testing and mocking framework for the Linux kernel.
+These tests are able to be run locally on a developer's workstation without a VM
+or special hardware.
+
+KUnit is heavily inspired by JUnit, Python's unittest.mock, and
+Googletest/Googlemock for C++. KUnit provides facilities for defining unit test
+cases, grouping related test cases into test suites, providing common
+infrastructure for running tests, and much more.
+
+Get started now: :doc:`start`
+
+Why KUnit?
+==========
+
+A unit test is supposed to test a single unit of code in isolation, hence the
+name. A unit test should be the finest granularity of testing and as such should
+allow all possible code paths to be tested in the code under test; this is only
+possible if the code under test is very small and does not have any external
+dependencies outside of the test's control like hardware.
+
+Outside of KUnit, there are no testing frameworks currently
+available for the kernel that do not require installing the kernel on a test
+machine or in a VM and all require tests to be written in userspace running on
+the kernel; this is true for Autotest, and kselftest, disqualifying
+any of them from being considered unit testing frameworks.
+
+KUnit addresses the problem of being able to run tests without needing a virtual
+machine or actual hardware with User Mode Linux. User Mode Linux is a Linux
+architecture, like ARM or x86; however, unlike other architectures it compiles
+to a standalone program that can be run like any other program directly inside
+of a host operating system; to be clear, it does not require any virtualization
+support; it is just a regular program.
+
+KUnit is fast. Excluding build time, from invocation to completion KUnit can run
+several dozen tests in only 10 to 20 seconds; this might not sound like a big
+deal to some people, but having such fast and easy to run tests fundamentally
+changes the way you go about testing and even writing code in the first place.
+Linus himself said in his `git talk at Google
+<https://gist.github.com/lorn/1272686/revisions#diff-53c65572127855f1b003db4064a94573R874>`_:
+
+ "... a lot of people seem to think that performance is about doing the
+ same thing, just doing it faster, and that is not true. That is not what
+ performance is all about. If you can do something really fast, really
+ well, people will start using it differently."
+
+In this context Linus was talking about branching and merging,
+but this point also applies to testing. If your tests are slow, unreliable, are
+difficult to write, and require a special setup or special hardware to run,
+then you wait a lot longer to write tests, and you wait a lot longer to run
+tests; this means that tests are likely to break, unlikely to test a lot of
+things, and are unlikely to be rerun once they pass. If your tests are really
+fast, you run them all the time, every time you make a change, and every time
+someone sends you some code. Why trust that someone ran all their tests
+correctly on every change when you can just run them yourself in less time than
+it takes to read his / her test log?
+
+How do I use it?
+================
+
+* :doc:`start` - for new users of KUnit
+* :doc:`usage` - for a more detailed explanation of KUnit features
+* :doc:`api/index` - for the list of KUnit APIs used for testing
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
new file mode 100644
index 0000000000000..6dc229e46bb34
--- /dev/null
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -0,0 +1,180 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===============
+Getting Started
+===============
+
+Installing dependencies
+=======================
+KUnit has the same dependencies as the Linux kernel. As long as you can build
+the kernel, you can run KUnit.
+
+KUnit Wrapper
+=============
+Included with KUnit is a simple Python wrapper that helps format the output to
+easily use and read KUnit output. It handles building and running the kernel, as
+well as formatting the output.
+
+The wrapper can be run with:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run
+
+Creating a kunitconfig
+======================
+The Python script is a thin wrapper around Kbuild as such, it needs to be
+configured with a ``kunitconfig`` file. This file essentially contains the
+regular Kernel config, with the specific test targets as well.
+
+.. code-block:: bash
+
+ git clone -b master https://kunit.googlesource.com/kunitconfig $PATH_TO_KUNITCONFIG_REPO
+ cd $PATH_TO_LINUX_REPO
+ ln -s $PATH_TO_KUNIT_CONFIG_REPO/kunitconfig kunitconfig
+
+You may want to add kunitconfig to your local gitignore.
+
+Verifying KUnit Works
+---------------------
+
+To make sure that everything is set up correctly, simply invoke the Python
+wrapper from your kernel repo:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py
+
+.. note::
+ You may want to run ``make mrproper`` first.
+
+If everything worked correctly, you should see the following:
+
+.. code-block:: bash
+
+ Generating .config ...
+ Building KUnit Kernel ...
+ Starting KUnit Kernel ...
+
+followed by a list of tests that are run. All of them should be passing.
+
+.. note::
+ Because it is building a lot of sources for the first time, the ``Building
+ kunit kernel`` step may take a while.
+
+Writing your first test
+=======================
+
+In your kernel repo let's add some code that we can test. Create a file
+``drivers/misc/example.h`` with the contents:
+
+.. code-block:: c
+
+ int misc_example_add(int left, int right);
+
+create a file ``drivers/misc/example.c``:
+
+.. code-block:: c
+
+ #include <linux/errno.h>
+
+ #include "example.h"
+
+ int misc_example_add(int left, int right)
+ {
+ return left + right;
+ }
+
+Now add the following lines to ``drivers/misc/Kconfig``:
+
+.. code-block:: kconfig
+
+ config MISC_EXAMPLE
+ bool "My example"
+
+and the following lines to ``drivers/misc/Makefile``:
+
+.. code-block:: make
+
+ obj-$(CONFIG_MISC_EXAMPLE) += example.o
+
+Now we are ready to write the test. The test will be in
+``drivers/misc/example-test.c``:
+
+.. code-block:: c
+
+ #include <kunit/test.h>
+ #include "example.h"
+
+ /* Define the test cases. */
+
+ static void misc_example_add_test_basic(struct kunit *test)
+ {
+ KUNIT_EXPECT_EQ(test, 1, misc_example_add(1, 0));
+ KUNIT_EXPECT_EQ(test, 2, misc_example_add(1, 1));
+ KUNIT_EXPECT_EQ(test, 0, misc_example_add(-1, 1));
+ KUNIT_EXPECT_EQ(test, INT_MAX, misc_example_add(0, INT_MAX));
+ KUNIT_EXPECT_EQ(test, -1, misc_example_add(INT_MAX, INT_MIN));
+ }
+
+ static void misc_example_test_failure(struct kunit *test)
+ {
+ KUNIT_FAIL(test, "This test never passes.");
+ }
+
+ static struct kunit_case misc_example_test_cases[] = {
+ KUNIT_CASE(misc_example_add_test_basic),
+ KUNIT_CASE(misc_example_test_failure),
+ {}
+ };
+
+ static struct kunit_suite misc_example_test_suite = {
+ .name = "misc-example",
+ .test_cases = misc_example_test_cases,
+ };
+ kunit_test_suite(misc_example_test_suite);
+
+Now add the following to ``drivers/misc/Kconfig``:
+
+.. code-block:: kconfig
+
+ config MISC_EXAMPLE_TEST
+ bool "Test for my example"
+ depends on MISC_EXAMPLE && KUNIT
+
+and the following to ``drivers/misc/Makefile``:
+
+.. code-block:: make
+
+ obj-$(CONFIG_MISC_EXAMPLE_TEST) += example-test.o
+
+Now add it to your ``kunitconfig``:
+
+.. code-block:: none
+
+ CONFIG_MISC_EXAMPLE=y
+ CONFIG_MISC_EXAMPLE_TEST=y
+
+Now you can run the test:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py
+
+You should see the following failure:
+
+.. code-block:: none
+
+ ...
+ [16:08:57] [PASSED] misc-example:misc_example_add_test_basic
+ [16:08:57] [FAILED] misc-example:misc_example_test_failure
+ [16:08:57] EXPECTATION FAILED at drivers/misc/example-test.c:17
+ [16:08:57] This test never passes.
+ ...
+
+Congrats! You just wrote your first KUnit test!
+
+Next Steps
+==========
+* Check out the :doc:`usage` page for a more
+ in-depth explanation of KUnit.
diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst
new file mode 100644
index 0000000000000..049886006dbce
--- /dev/null
+++ b/Documentation/dev-tools/kunit/usage.rst
@@ -0,0 +1,575 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===========
+Using KUnit
+===========
+
+The purpose of this document is to describe what KUnit is, how it works, how it
+is intended to be used, and all the concepts and terminology that are needed to
+understand it. This guide assumes a working knowledge of the Linux kernel and
+some basic knowledge of testing.
+
+For a high level introduction to KUnit, including setting up KUnit for your
+project, see :doc:`start`.
+
+Organization of this document
+=============================
+
+This document is organized into two main sections: Testing and Isolating
+Behavior. The first covers what a unit test is and how to use KUnit to write
+them. The second covers how to use KUnit to isolate code and make it possible
+to unit test code that was otherwise un-unit-testable.
+
+Testing
+=======
+
+What is KUnit?
+--------------
+
+"K" is short for "kernel" so "KUnit" is the "(Linux) Kernel Unit Testing
+Framework." KUnit is intended first and foremost for writing unit tests; it is
+general enough that it can be used to write integration tests; however, this is
+a secondary goal. KUnit has no ambition of being the only testing framework for
+the kernel; for example, it does not intend to be an end-to-end testing
+framework.
+
+What is Unit Testing?
+---------------------
+
+A `unit test <https://martinfowler.com/bliki/UnitTest.html>`_ is a test that
+tests code at the smallest possible scope, a *unit* of code. In the C
+programming language that's a function.
+
+Unit tests should be written for all the publicly exposed functions in a
+compilation unit; so that is all the functions that are exported in either a
+*class* (defined below) or all functions which are **not** static.
+
+Writing Tests
+-------------
+
+Test Cases
+~~~~~~~~~~
+
+The fundamental unit in KUnit is the test case. A test case is a function with
+the signature ``void (*)(struct kunit *test)``. It calls a function to be tested
+and then sets *expectations* for what should happen. For example:
+
+.. code-block:: c
+
+ void example_test_success(struct kunit *test)
+ {
+ }
+
+ void example_test_failure(struct kunit *test)
+ {
+ KUNIT_FAIL(test, "This test never passes.");
+ }
+
+In the above example ``example_test_success`` always passes because it does
+nothing; no expectations are set, so all expectations pass. On the other hand
+``example_test_failure`` always fails because it calls ``KUNIT_FAIL``, which is
+a special expectation that logs a message and causes the test case to fail.
+
+Expectations
+~~~~~~~~~~~~
+An *expectation* is a way to specify that you expect a piece of code to do
+something in a test. An expectation is called like a function. A test is made
+by setting expectations about the behavior of a piece of code under test; when
+one or more of the expectations fail, the test case fails and information about
+the failure is logged. For example:
+
+.. code-block:: c
+
+ void add_test_basic(struct kunit *test)
+ {
+ KUNIT_EXPECT_EQ(test, 1, add(1, 0));
+ KUNIT_EXPECT_EQ(test, 2, add(1, 1));
+ }
+
+In the above example ``add_test_basic`` makes a number of assertions about the
+behavior of a function called ``add``; the first parameter is always of type
+``struct kunit *``, which contains information about the current test context;
+the second parameter, in this case, is what the value is expected to be; the
+last value is what the value actually is. If ``add`` passes all of these
+expectations, the test case, ``add_test_basic`` will pass; if any one of these
+expectations fail, the test case will fail.
+
+It is important to understand that a test case *fails* when any expectation is
+violated; however, the test will continue running, potentially trying other
+expectations until the test case ends or is otherwise terminated. This is as
+opposed to *assertions* which are discussed later.
+
+To learn about more expectations supported by KUnit, see :doc:`api/test`.
+
+.. note::
+ A single test case should be pretty short, pretty easy to understand,
+ focused on a single behavior.
+
+For example, if we wanted to properly test the add function above, we would
+create additional tests cases which would each test a different property that an
+add function should have like this:
+
+.. code-block:: c
+
+ void add_test_basic(struct kunit *test)
+ {
+ KUNIT_EXPECT_EQ(test, 1, add(1, 0));
+ KUNIT_EXPECT_EQ(test, 2, add(1, 1));
+ }
+
+ void add_test_negative(struct kunit *test)
+ {
+ KUNIT_EXPECT_EQ(test, 0, add(-1, 1));
+ }
+
+ void add_test_max(struct kunit *test)
+ {
+ KUNIT_EXPECT_EQ(test, INT_MAX, add(0, INT_MAX));
+ KUNIT_EXPECT_EQ(test, -1, add(INT_MAX, INT_MIN));
+ }
+
+ void add_test_overflow(struct kunit *test)
+ {
+ KUNIT_EXPECT_EQ(test, INT_MIN, add(INT_MAX, 1));
+ }
+
+Notice how it is immediately obvious what all the properties that we are testing
+for are.
+
+Assertions
+~~~~~~~~~~
+
+KUnit also has the concept of an *assertion*. An assertion is just like an
+expectation except the assertion immediately terminates the test case if it is
+not satisfied.
+
+For example:
+
+.. code-block:: c
+
+ static void mock_test_do_expect_default_return(struct kunit *test)
+ {
+ struct mock_test_context *ctx = test->priv;
+ struct mock *mock = ctx->mock;
+ int param0 = 5, param1 = -5;
+ const char *two_param_types[] = {"int", "int"};
+ const void *two_params[] = {¶m0, ¶m1};
+ const void *ret;
+
+ ret = mock->do_expect(mock,
+ "test_printk", test_printk,
+ two_param_types, two_params,
+ ARRAY_SIZE(two_params));
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ret);
+ KUNIT_EXPECT_EQ(test, -4, *((int *) ret));
+ }
+
+In this example, the method under test should return a pointer to a value, so
+if the pointer returned by the method is null or an errno, we don't want to
+bother continuing the test since the following expectation could crash the test
+case. `ASSERT_NOT_ERR_OR_NULL(...)` allows us to bail out of the test case if
+the appropriate conditions have not been satisfied to complete the test.
+
+Test Suites
+~~~~~~~~~~~
+
+Now obviously one unit test isn't very helpful; the power comes from having
+many test cases covering all of your behaviors. Consequently it is common to
+have many *similar* tests; in order to reduce duplication in these closely
+related tests most unit testing frameworks provide the concept of a *test
+suite*, in KUnit we call it a *test suite*; all it is is just a collection of
+test cases for a unit of code with a set up function that gets invoked before
+every test cases and then a tear down function that gets invoked after every
+test case completes.
+
+Example:
+
+.. code-block:: c
+
+ static struct kunit_case example_test_cases[] = {
+ KUNIT_CASE(example_test_foo),
+ KUNIT_CASE(example_test_bar),
+ KUNIT_CASE(example_test_baz),
+ {}
+ };
+
+ static struct kunit_suite example_test_suite = {
+ .name = "example",
+ .init = example_test_init,
+ .exit = example_test_exit,
+ .test_cases = example_test_cases,
+ };
+ kunit_test_suite(example_test_suite);
+
+In the above example the test suite, ``example_test_suite``, would run the test
+cases ``example_test_foo``, ``example_test_bar``, and ``example_test_baz``,
+each would have ``example_test_init`` called immediately before it and would
+have ``example_test_exit`` called immediately after it.
+``kunit_test_suite(example_test_suite)`` registers the test suite with the
+KUnit test framework.
+
+.. note::
+ A test case will only be run if it is associated with a test suite.
+
+For a more information on these types of things see the :doc:`api/test`.
+
+Isolating Behavior
+==================
+
+The most important aspect of unit testing that other forms of testing do not
+provide is the ability to limit the amount of code under test to a single unit.
+In practice, this is only possible by being able to control what code gets run
+when the unit under test calls a function and this is usually accomplished
+through some sort of indirection where a function is exposed as part of an API
+such that the definition of that function can be changed without affecting the
+rest of the code base. In the kernel this primarily comes from two constructs,
+classes, structs that contain function pointers that are provided by the
+implementer, and architecture specific functions which have definitions selected
+at compile time.
+
+Classes
+-------
+
+Classes are not a construct that is built into the C programming language;
+however, it is an easily derived concept. Accordingly, pretty much every project
+that does not use a standardized object oriented library (like GNOME's GObject)
+has their own slightly different way of doing object oriented programming; the
+Linux kernel is no exception.
+
+The central concept in kernel object oriented programming is the class. In the
+kernel, a *class* is a struct that contains function pointers. This creates a
+contract between *implementers* and *users* since it forces them to use the
+same function signature without having to call the function directly. In order
+for it to truly be a class, the function pointers must specify that a pointer
+to the class, known as a *class handle*, be one of the parameters; this makes
+it possible for the member functions (also known as *methods*) to have access
+to member variables (more commonly known as *fields*) allowing the same
+implementation to have multiple *instances*.
+
+Typically a class can be *overridden* by *child classes* by embedding the
+*parent class* in the child class. Then when a method provided by the child
+class is called, the child implementation knows that the pointer passed to it is
+of a parent contained within the child; because of this, the child can compute
+the pointer to itself because the pointer to the parent is always a fixed offset
+from the pointer to the child; this offset is the offset of the parent contained
+in the child struct. For example:
+
+.. code-block:: c
+
+ struct shape {
+ int (*area)(struct shape *this);
+ };
+
+ struct rectangle {
+ struct shape parent;
+ int length;
+ int width;
+ };
+
+ int rectangle_area(struct shape *this)
+ {
+ struct rectangle *self = container_of(this, struct shape, parent);
+
+ return self->length * self->width;
+ };
+
+ void rectangle_new(struct rectangle *self, int length, int width)
+ {
+ self->parent.area = rectangle_area;
+ self->length = length;
+ self->width = width;
+ }
+
+In this example (as in most kernel code) the operation of computing the pointer
+to the child from the pointer to the parent is done by ``container_of``.
+
+Faking Classes
+~~~~~~~~~~~~~~
+
+In order to unit test a piece of code that calls a method in a class, the
+behavior of the method must be controllable, otherwise the test ceases to be a
+unit test and becomes an integration test.
+
+A fake just provides an implementation of a piece of code that is different than
+what runs in a production instance, but behaves identically from the standpoint
+of the callers; this is usually done to replace a dependency that is hard to
+deal with, or is slow.
+
+A good example for this might be implementing a fake EEPROM that just stores the
+"contents" in an internal buffer. For example, let's assume we have a class that
+represents an EEPROM:
+
+.. code-block:: c
+
+ struct eeprom {
+ ssize_t (*read)(struct eeprom *this, size_t offset, char *buffer, size_t count);
+ ssize_t (*write)(struct eeprom *this, size_t offset, const char *buffer, size_t count);
+ };
+
+And we want to test some code that buffers writes to the EEPROM:
+
+.. code-block:: c
+
+ struct eeprom_buffer {
+ ssize_t (*write)(struct eeprom_buffer *this, const char *buffer, size_t count);
+ int flush(struct eeprom_buffer *this);
+ size_t flush_count; /* Flushes when buffer exceeds flush_count. */
+ };
+
+ struct eeprom_buffer *new_eeprom_buffer(struct eeprom *eeprom);
+ void destroy_eeprom_buffer(struct eeprom *eeprom);
+
+We can easily test this code by *faking out* the underlying EEPROM:
+
+.. code-block:: c
+
+ struct fake_eeprom {
+ struct eeprom parent;
+ char contents[FAKE_EEPROM_CONTENTS_SIZE];
+ };
+
+ ssize_t fake_eeprom_read(struct eeprom *parent, size_t offset, char *buffer, size_t count)
+ {
+ struct fake_eeprom *this = container_of(parent, struct fake_eeprom, parent);
+
+ count = min(count, FAKE_EEPROM_CONTENTS_SIZE - offset);
+ memcpy(buffer, this->contents + offset, count);
+
+ return count;
+ }
+
+ ssize_t fake_eeprom_write(struct eeprom *this, size_t offset, const char *buffer, size_t count)
+ {
+ struct fake_eeprom *this = container_of(parent, struct fake_eeprom, parent);
+
+ count = min(count, FAKE_EEPROM_CONTENTS_SIZE - offset);
+ memcpy(this->contents + offset, buffer, count);
+
+ return count;
+ }
+
+ void fake_eeprom_init(struct fake_eeprom *this)
+ {
+ this->parent.read = fake_eeprom_read;
+ this->parent.write = fake_eeprom_write;
+ memset(this->contents, 0, FAKE_EEPROM_CONTENTS_SIZE);
+ }
+
+We can now use it to test ``struct eeprom_buffer``:
+
+.. code-block:: c
+
+ struct eeprom_buffer_test {
+ struct fake_eeprom *fake_eeprom;
+ struct eeprom_buffer *eeprom_buffer;
+ };
+
+ static void eeprom_buffer_test_does_not_write_until_flush(struct kunit *test)
+ {
+ struct eeprom_buffer_test *ctx = test->priv;
+ struct eeprom_buffer *eeprom_buffer = ctx->eeprom_buffer;
+ struct fake_eeprom *fake_eeprom = ctx->fake_eeprom;
+ char buffer[] = {0xff};
+
+ eeprom_buffer->flush_count = SIZE_MAX;
+
+ eeprom_buffer->write(eeprom_buffer, buffer, 1);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[0], 0);
+
+ eeprom_buffer->write(eeprom_buffer, buffer, 1);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[1], 0);
+
+ eeprom_buffer->flush(eeprom_buffer);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[0], 0xff);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[1], 0xff);
+ }
+
+ static void eeprom_buffer_test_flushes_after_flush_count_met(struct kunit *test)
+ {
+ struct eeprom_buffer_test *ctx = test->priv;
+ struct eeprom_buffer *eeprom_buffer = ctx->eeprom_buffer;
+ struct fake_eeprom *fake_eeprom = ctx->fake_eeprom;
+ char buffer[] = {0xff};
+
+ eeprom_buffer->flush_count = 2;
+
+ eeprom_buffer->write(eeprom_buffer, buffer, 1);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[0], 0);
+
+ eeprom_buffer->write(eeprom_buffer, buffer, 1);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[0], 0xff);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[1], 0xff);
+ }
+
+ static void eeprom_buffer_test_flushes_increments_of_flush_count(struct kunit *test)
+ {
+ struct eeprom_buffer_test *ctx = test->priv;
+ struct eeprom_buffer *eeprom_buffer = ctx->eeprom_buffer;
+ struct fake_eeprom *fake_eeprom = ctx->fake_eeprom;
+ char buffer[] = {0xff, 0xff};
+
+ eeprom_buffer->flush_count = 2;
+
+ eeprom_buffer->write(eeprom_buffer, buffer, 1);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[0], 0);
+
+ eeprom_buffer->write(eeprom_buffer, buffer, 2);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[0], 0xff);
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[1], 0xff);
+ /* Should have only flushed the first two bytes. */
+ KUNIT_EXPECT_EQ(test, fake_eeprom->contents[2], 0);
+ }
+
+ static int eeprom_buffer_test_init(struct kunit *test)
+ {
+ struct eeprom_buffer_test *ctx;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+ ctx->fake_eeprom = kunit_kzalloc(test, sizeof(*ctx->fake_eeprom), GFP_KERNEL);
+ ASSERT_NOT_ERR_OR_NULL(test, ctx->fake_eeprom);
+
+ ctx->eeprom_buffer = new_eeprom_buffer(&ctx->fake_eeprom->parent);
+ ASSERT_NOT_ERR_OR_NULL(test, ctx->eeprom_buffer);
+
+ test->priv = ctx;
+
+ return 0;
+ }
+
+ static void eeprom_buffer_test_exit(struct kunit *test)
+ {
+ struct eeprom_buffer_test *ctx = test->priv;
+
+ destroy_eeprom_buffer(ctx->eeprom_buffer);
+ }
+
+.. _kunit-on-non-uml:
+
+KUnit on non-UML architectures
+==============================
+
+By default KUnit uses UML as a way to provide dependencies for code under test.
+Under most circumstances KUnit's usage of UML should be treated as an
+implementation detail of how KUnit works under the hood. Nevertheless, there
+are instances where being able to run architecture specific code, or test
+against real hardware is desirable. For these reasons KUnit supports running on
+other architectures.
+
+Running existing KUnit tests on non-UML architectures
+-----------------------------------------------------
+
+There are some special considerations when running existing KUnit tests on
+non-UML architectures:
+
+* Hardware may not be deterministic, so a test that always passes or fails
+ when run under UML may not always do so on real hardware.
+* Hardware and VM environments may not be hermetic. KUnit tries its best to
+ provide a hermetic environment to run tests; however, it cannot manage state
+ that it doesn't know about outside of the kernel. Consequently, tests that
+ may be hermetic on UML may not be hermetic on other architectures.
+* Some features and tooling may not be supported outside of UML.
+* Hardware and VMs are slower than UML.
+
+None of these are reasons not to run your KUnit tests on real hardware; they are
+only things to be aware of when doing so.
+
+The biggest impediment will likely be that certain KUnit features and
+infrastructure may not support your target environment. For example, at this
+time the KUnit Wrapper (``tools/testing/kunit/kunit.py``) does not work outside
+of UML. Unfortunately, there is no way around this. Using UML (or even just a
+particular architecture) allows us to make a lot of assumptions that make it
+possible to do things which might otherwise be impossible.
+
+Nevertheless, all core KUnit framework features are fully supported on all
+architectures, and using them is straightforward: all you need to do is to take
+your kunitconfig, your Kconfig options for the tests you would like to run, and
+merge them into whatever config your are using for your platform. That's it!
+
+For example, let's say you have the following kunitconfig:
+
+.. code-block:: none
+
+ CONFIG_KUNIT=y
+ CONFIG_KUNIT_EXAMPLE_TEST=y
+
+If you wanted to run this test on an x86 VM, you might add the following config
+options to your ``.config``:
+
+.. code-block:: none
+
+ CONFIG_KUNIT=y
+ CONFIG_KUNIT_EXAMPLE_TEST=y
+ CONFIG_SERIAL_8250=y
+ CONFIG_SERIAL_8250_CONSOLE=y
+
+All these new options do is enable support for a common serial console needed
+for logging.
+
+Next, you could build a kernel with these tests as follows:
+
+
+.. code-block:: bash
+
+ make ARCH=x86 olddefconfig
+ make ARCH=x86
+
+Once you have built a kernel, you could run it on QEMU as follows:
+
+.. code-block:: bash
+
+ qemu-system-x86_64 -enable-kvm \
+ -m 1024 \
+ -kernel arch/x86_64/boot/bzImage \
+ -append 'console=ttyS0' \
+ --nographic
+
+Interspersed in the kernel logs you might see the following:
+
+.. code-block:: none
+
+ TAP version 14
+ # Subtest: example
+ 1..1
+ # example_simple_test: initializing
+ ok 1 - example_simple_test
+ ok 1 - example
+
+Congratulations, you just ran a KUnit test on the x86 architecture!
+
+Writing new tests for other architectures
+-----------------------------------------
+
+The first thing you must do is ask yourself whether it is necessary to write a
+KUnit test for a specific architecture, and then whether it is necessary to
+write that test for a particular piece of hardware. In general, writing a test
+that depends on having access to a particular piece of hardware or software (not
+included in the Linux source repo) should be avoided at all costs.
+
+Even if you only ever plan on running your KUnit test on your hardware
+configuration, other people may want to run your tests and may not have access
+to your hardware. If you write your test to run on UML, then anyone can run your
+tests without knowing anything about your particular setup, and you can still
+run your tests on your hardware setup just by compiling for your architecture.
+
+.. important::
+ Always prefer tests that run on UML to tests that only run under a particular
+ architecture, and always prefer tests that run under QEMU or another easy
+ (and monitarily free) to obtain software environment to a specific piece of
+ hardware.
+
+Nevertheless, there are still valid reasons to write an architecture or hardware
+specific test: for example, you might want to test some code that really belongs
+in ``arch/some-arch/*``. Even so, try your best to write the test so that it
+does not depend on physical hardware: if some of your test cases don't need the
+hardware, only require the hardware for tests that actually need it.
+
+Now that you have narrowed down exactly what bits are hardware specific, the
+actual procedure for writing and running the tests is pretty much the same as
+writing normal KUnit tests. One special caveat is that you have to reset
+hardware state in between test cases; if this is not possible, you may only be
+able to run one test case per invocation.
+
+.. TODO(brendanhiggins@google.com): Add an actual example of an architecture
+ dependent KUnit test.
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 16/18] MAINTAINERS: add entry for KUnit the unit testing framework
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Brendan Higgins
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
Add myself as maintainer of KUnit, the Linux kernel's unit testing
framework.
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
---
MAINTAINERS | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 677ef41cb012c..48d04d180a988 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -8599,6 +8599,17 @@ S: Maintained
F: tools/testing/selftests/
F: Documentation/dev-tools/kselftest*
+KERNEL UNIT TESTING FRAMEWORK (KUnit)
+M: Brendan Higgins <brendanhiggins@google.com>
+L: linux-kselftest@vger.kernel.org
+L: kunit-dev@googlegroups.com
+W: https://google.github.io/kunit-docs/third_party/kernel/docs/
+S: Maintained
+F: Documentation/dev-tools/kunit/
+F: include/kunit/
+F: kunit/
+F: tools/testing/kunit/
+
KERNEL USERMODE HELPER
M: Luis Chamberlain <mcgrof@kernel.org>
L: linux-kernel@vger.kernel.org
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 17/18] kernel/sysctl-test: Add null pointer test for sysctl.c:proc_dointvec()
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Iurii Zaikin, Brendan Higgins
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
From: Iurii Zaikin <yzaikin@google.com>
KUnit tests for initialized data behavior of proc_dointvec that is
explicitly checked in the code. Includes basic parsing tests including
int min/max overflow.
Signed-off-by: Iurii Zaikin <yzaikin@google.com>
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
Acked-by: Luis Chamberlain <mcgrof@kernel.org>
---
kernel/Makefile | 2 +
kernel/sysctl-test.c | 392 +++++++++++++++++++++++++++++++++++++++++++
lib/Kconfig.debug | 11 ++
3 files changed, 405 insertions(+)
create mode 100644 kernel/sysctl-test.c
diff --git a/kernel/Makefile b/kernel/Makefile
index a8d923b5481ba..50fd511cd0ee0 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -114,6 +114,8 @@ obj-$(CONFIG_HAS_IOMEM) += iomem.o
obj-$(CONFIG_ZONE_DEVICE) += memremap.o
obj-$(CONFIG_RSEQ) += rseq.o
+obj-$(CONFIG_SYSCTL_KUNIT_TEST) += sysctl-test.o
+
obj-$(CONFIG_GCC_PLUGIN_STACKLEAK) += stackleak.o
KASAN_SANITIZE_stackleak.o := n
KCOV_INSTRUMENT_stackleak.o := n
diff --git a/kernel/sysctl-test.c b/kernel/sysctl-test.c
new file mode 100644
index 0000000000000..2a63241a8453b
--- /dev/null
+++ b/kernel/sysctl-test.c
@@ -0,0 +1,392 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit test of proc sysctl.
+ */
+
+#include <kunit/test.h>
+#include <linux/sysctl.h>
+
+#define KUNIT_PROC_READ 0
+#define KUNIT_PROC_WRITE 1
+
+static int i_zero;
+static int i_one_hundred = 100;
+
+/*
+ * Test that proc_dointvec will not try to use a NULL .data field even when the
+ * length is non-zero.
+ */
+static void sysctl_test_api_dointvec_null_tbl_data(struct kunit *test)
+{
+ struct ctl_table null_data_table = {
+ .procname = "foo",
+ /*
+ * Here we are testing that proc_dointvec behaves correctly when
+ * we give it a NULL .data field. Normally this would point to a
+ * piece of memory where the value would be stored.
+ */
+ .data = NULL,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ /*
+ * proc_dointvec expects a buffer in user space, so we allocate one. We
+ * also need to cast it to __user so sparse doesn't get mad.
+ */
+ void __user *buffer = (void __user *)kunit_kzalloc(test, sizeof(int),
+ GFP_USER);
+ size_t len;
+ loff_t pos;
+
+ /*
+ * We don't care what the starting length is since proc_dointvec should
+ * not try to read because .data is NULL.
+ */
+ len = 1234;
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&null_data_table,
+ KUNIT_PROC_READ, buffer, &len,
+ &pos));
+ KUNIT_EXPECT_EQ(test, (size_t)0, len);
+
+ /*
+ * See above.
+ */
+ len = 1234;
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&null_data_table,
+ KUNIT_PROC_WRITE, buffer, &len,
+ &pos));
+ KUNIT_EXPECT_EQ(test, (size_t)0, len);
+}
+
+/*
+ * Similar to the previous test, we create a struct ctrl_table that has a .data
+ * field that proc_dointvec cannot do anything with; however, this time it is
+ * because we tell proc_dointvec that the size is 0.
+ */
+static void sysctl_test_api_dointvec_table_maxlen_unset(struct kunit *test)
+{
+ int data = 0;
+ struct ctl_table data_maxlen_unset_table = {
+ .procname = "foo",
+ .data = &data,
+ /*
+ * So .data is no longer NULL, but we tell proc_dointvec its
+ * length is 0, so it still shouldn't try to use it.
+ */
+ .maxlen = 0,
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ void __user *buffer = (void __user *)kunit_kzalloc(test, sizeof(int),
+ GFP_USER);
+ size_t len;
+ loff_t pos;
+
+ /*
+ * As before, we don't care what buffer length is because proc_dointvec
+ * cannot do anything because its internal .data buffer has zero length.
+ */
+ len = 1234;
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&data_maxlen_unset_table,
+ KUNIT_PROC_READ, buffer, &len,
+ &pos));
+ KUNIT_EXPECT_EQ(test, (size_t)0, len);
+
+ /*
+ * See previous comment.
+ */
+ len = 1234;
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&data_maxlen_unset_table,
+ KUNIT_PROC_WRITE, buffer, &len,
+ &pos));
+ KUNIT_EXPECT_EQ(test, (size_t)0, len);
+}
+
+/*
+ * Here we provide a valid struct ctl_table, but we try to read and write from
+ * it using a buffer of zero length, so it should still fail in a similar way as
+ * before.
+ */
+static void sysctl_test_api_dointvec_table_len_is_zero(struct kunit *test)
+{
+ int data = 0;
+ /* Good table. */
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ void __user *buffer = (void __user *)kunit_kzalloc(test, sizeof(int),
+ GFP_USER);
+ /*
+ * However, now our read/write buffer has zero length.
+ */
+ size_t len = 0;
+ loff_t pos;
+
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&table, KUNIT_PROC_READ, buffer,
+ &len, &pos));
+ KUNIT_EXPECT_EQ(test, (size_t)0, len);
+
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&table, KUNIT_PROC_WRITE, buffer,
+ &len, &pos));
+ KUNIT_EXPECT_EQ(test, (size_t)0, len);
+}
+
+/*
+ * Test that proc_dointvec refuses to read when the file position is non-zero.
+ */
+static void sysctl_test_api_dointvec_table_read_but_position_set(
+ struct kunit *test)
+{
+ int data = 0;
+ /* Good table. */
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ void __user *buffer = (void __user *)kunit_kzalloc(test, sizeof(int),
+ GFP_USER);
+ /*
+ * We don't care about our buffer length because we start off with a
+ * non-zero file position.
+ */
+ size_t len = 1234;
+ /*
+ * proc_dointvec should refuse to read into the buffer since the file
+ * pos is non-zero.
+ */
+ loff_t pos = 1;
+
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&table, KUNIT_PROC_READ, buffer,
+ &len, &pos));
+ KUNIT_EXPECT_EQ(test, (size_t)0, len);
+}
+
+/*
+ * Test that we can read a two digit number in a sufficiently size buffer.
+ * Nothing fancy.
+ */
+static void sysctl_test_dointvec_read_happy_single_positive(struct kunit *test)
+{
+ int data = 0;
+ /* Good table. */
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ size_t len = 4;
+ loff_t pos = 0;
+ char *buffer = kunit_kzalloc(test, len, GFP_USER);
+ char __user *user_buffer = (char __user *)buffer;
+ /* Store 13 in the data field. */
+ *((int *)table.data) = 13;
+
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&table, KUNIT_PROC_READ,
+ user_buffer, &len, &pos));
+ KUNIT_ASSERT_EQ(test, (size_t)3, len);
+ buffer[len] = '\0';
+ /* And we read 13 back out. */
+ KUNIT_EXPECT_STREQ(test, "13\n", buffer);
+}
+
+/*
+ * Same as previous test, just now with negative numbers.
+ */
+static void sysctl_test_dointvec_read_happy_single_negative(struct kunit *test)
+{
+ int data = 0;
+ /* Good table. */
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ size_t len = 5;
+ loff_t pos = 0;
+ char *buffer = kunit_kzalloc(test, len, GFP_USER);
+ char __user *user_buffer = (char __user *)buffer;
+ *((int *)table.data) = -16;
+
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&table, KUNIT_PROC_READ,
+ user_buffer, &len, &pos));
+ KUNIT_ASSERT_EQ(test, (size_t)4, len);
+ buffer[len] = '\0';
+ KUNIT_EXPECT_STREQ(test, "-16\n", (char *)buffer);
+}
+
+/*
+ * Test that a simple positive write works.
+ */
+static void sysctl_test_dointvec_write_happy_single_positive(struct kunit *test)
+{
+ int data = 0;
+ /* Good table. */
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ char input[] = "9";
+ size_t len = sizeof(input) - 1;
+ loff_t pos = 0;
+ char *buffer = kunit_kzalloc(test, len, GFP_USER);
+ char __user *user_buffer = (char __user *)buffer;
+
+ memcpy(buffer, input, len);
+
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&table, KUNIT_PROC_WRITE,
+ user_buffer, &len, &pos));
+ KUNIT_EXPECT_EQ(test, sizeof(input) - 1, len);
+ KUNIT_EXPECT_EQ(test, sizeof(input) - 1, (size_t)pos);
+ KUNIT_EXPECT_EQ(test, 9, *((int *)table.data));
+}
+
+/*
+ * Same as previous test, but now with negative numbers.
+ */
+static void sysctl_test_dointvec_write_happy_single_negative(struct kunit *test)
+{
+ int data = 0;
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ char input[] = "-9";
+ size_t len = sizeof(input) - 1;
+ loff_t pos = 0;
+ char *buffer = kunit_kzalloc(test, len, GFP_USER);
+ char __user *user_buffer = (char __user *)buffer;
+
+ memcpy(buffer, input, len);
+
+ KUNIT_EXPECT_EQ(test, 0, proc_dointvec(&table, KUNIT_PROC_WRITE,
+ user_buffer, &len, &pos));
+ KUNIT_EXPECT_EQ(test, sizeof(input) - 1, len);
+ KUNIT_EXPECT_EQ(test, sizeof(input) - 1, (size_t)pos);
+ KUNIT_EXPECT_EQ(test, -9, *((int *)table.data));
+}
+
+/*
+ * Test that writing a value smaller than the minimum possible value is not
+ * allowed.
+ */
+static void sysctl_test_api_dointvec_write_single_less_int_min(
+ struct kunit *test)
+{
+ int data = 0;
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ size_t max_len = 32, len = max_len;
+ loff_t pos = 0;
+ char *buffer = kunit_kzalloc(test, max_len, GFP_USER);
+ char __user *user_buffer = (char __user *)buffer;
+ unsigned long abs_of_less_than_min = (unsigned long)INT_MAX
+ - (INT_MAX + INT_MIN) + 1;
+
+ /*
+ * We use this rigmarole to create a string that contains a value one
+ * less than the minimum accepted value.
+ */
+ KUNIT_ASSERT_LT(test,
+ (size_t)snprintf(buffer, max_len, "-%lu",
+ abs_of_less_than_min),
+ max_len);
+
+ KUNIT_EXPECT_EQ(test, -EINVAL, proc_dointvec(&table, KUNIT_PROC_WRITE,
+ user_buffer, &len, &pos));
+ KUNIT_EXPECT_EQ(test, max_len, len);
+ KUNIT_EXPECT_EQ(test, 0, *((int *)table.data));
+}
+
+/*
+ * Test that writing the maximum possible value works.
+ */
+static void sysctl_test_api_dointvec_write_single_greater_int_max(
+ struct kunit *test)
+{
+ int data = 0;
+ struct ctl_table table = {
+ .procname = "foo",
+ .data = &data,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ };
+ size_t max_len = 32, len = max_len;
+ loff_t pos = 0;
+ char *buffer = kunit_kzalloc(test, max_len, GFP_USER);
+ char __user *user_buffer = (char __user *)buffer;
+ unsigned long greater_than_max = (unsigned long)INT_MAX + 1;
+
+ KUNIT_ASSERT_GT(test, greater_than_max, (unsigned long)INT_MAX);
+ KUNIT_ASSERT_LT(test, (size_t)snprintf(buffer, max_len, "%lu",
+ greater_than_max),
+ max_len);
+ KUNIT_EXPECT_EQ(test, -EINVAL, proc_dointvec(&table, KUNIT_PROC_WRITE,
+ user_buffer, &len, &pos));
+ KUNIT_ASSERT_EQ(test, max_len, len);
+ KUNIT_EXPECT_EQ(test, 0, *((int *)table.data));
+}
+
+static struct kunit_case sysctl_test_cases[] = {
+ KUNIT_CASE(sysctl_test_api_dointvec_null_tbl_data),
+ KUNIT_CASE(sysctl_test_api_dointvec_table_maxlen_unset),
+ KUNIT_CASE(sysctl_test_api_dointvec_table_len_is_zero),
+ KUNIT_CASE(sysctl_test_api_dointvec_table_read_but_position_set),
+ KUNIT_CASE(sysctl_test_dointvec_read_happy_single_positive),
+ KUNIT_CASE(sysctl_test_dointvec_read_happy_single_negative),
+ KUNIT_CASE(sysctl_test_dointvec_write_happy_single_positive),
+ KUNIT_CASE(sysctl_test_dointvec_write_happy_single_negative),
+ KUNIT_CASE(sysctl_test_api_dointvec_write_single_less_int_min),
+ KUNIT_CASE(sysctl_test_api_dointvec_write_single_greater_int_max),
+ {}
+};
+
+static struct kunit_suite sysctl_test_suite = {
+ .name = "sysctl_test",
+ .test_cases = sysctl_test_cases,
+};
+
+kunit_test_suite(sysctl_test_suite);
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index cbdfae3798965..6f8007800a76f 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1939,6 +1939,17 @@ config TEST_SYSCTL
If unsure, say N.
+config SYSCTL_KUNIT_TEST
+ bool "KUnit test for sysctl"
+ depends on KUNIT
+ help
+ This builds the proc sysctl unit test, which runs on boot.
+ Tests the API contract and implementation correctness of sysctl.
+ For more information on KUnit and unit tests in general please refer
+ to the KUnit documentation in Documentation/dev-tools/kunit/.
+
+ If unsure, say N.
+
config TEST_UDELAY
tristate "udelay test driver"
help
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 18/18] MAINTAINERS: add proc sysctl KUnit test to PROC SYSCTL section
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Brendan Higgins, Iurii Zaikin
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
Add entry for the new proc sysctl KUnit test to the PROC SYSCTL section,
and add Iurii as a maintainer.
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Cc: Iurii Zaikin <yzaikin@google.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
Acked-by: Luis Chamberlain <mcgrof@kernel.org>
---
MAINTAINERS | 2 ++
1 file changed, 2 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 48d04d180a988..f8204c75114da 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -12721,12 +12721,14 @@ F: Documentation/filesystems/proc.txt
PROC SYSCTL
M: Luis Chamberlain <mcgrof@kernel.org>
M: Kees Cook <keescook@chromium.org>
+M: Iurii Zaikin <yzaikin@google.com>
L: linux-kernel@vger.kernel.org
L: linux-fsdevel@vger.kernel.org
S: Maintained
F: fs/proc/proc_sysctl.c
F: include/linux/sysctl.h
F: kernel/sysctl.c
+F: kernel/sysctl-test.c
F: tools/testing/selftests/sysctl/
PS3 NETWORK SUPPORT
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 14/18] kunit: defconfig: add defconfigs for building KUnit tests
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg,
Brendan Higgins
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
Add defconfig for UML and a fragment that can be used to configure other
architectures for building KUnit tests. Add option to kunit_tool to use
a defconfig to create the kunitconfig.
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
---
arch/um/configs/kunit_defconfig | 8 ++++++++
tools/testing/kunit/configs/all_tests.config | 8 ++++++++
tools/testing/kunit/kunit.py | 18 ++++++++++++++++--
tools/testing/kunit/kunit_kernel.py | 3 ++-
4 files changed, 34 insertions(+), 3 deletions(-)
create mode 100644 arch/um/configs/kunit_defconfig
create mode 100644 tools/testing/kunit/configs/all_tests.config
diff --git a/arch/um/configs/kunit_defconfig b/arch/um/configs/kunit_defconfig
new file mode 100644
index 0000000000000..bfe49689038f1
--- /dev/null
+++ b/arch/um/configs/kunit_defconfig
@@ -0,0 +1,8 @@
+CONFIG_OF=y
+CONFIG_OF_UNITTEST=y
+CONFIG_OF_OVERLAY=y
+CONFIG_I2C=y
+CONFIG_I2C_MUX=y
+CONFIG_KUNIT=y
+CONFIG_KUNIT_TEST=y
+CONFIG_KUNIT_EXAMPLE_TEST=y
diff --git a/tools/testing/kunit/configs/all_tests.config b/tools/testing/kunit/configs/all_tests.config
new file mode 100644
index 0000000000000..bfe49689038f1
--- /dev/null
+++ b/tools/testing/kunit/configs/all_tests.config
@@ -0,0 +1,8 @@
+CONFIG_OF=y
+CONFIG_OF_UNITTEST=y
+CONFIG_OF_OVERLAY=y
+CONFIG_I2C=y
+CONFIG_I2C_MUX=y
+CONFIG_KUNIT=y
+CONFIG_KUNIT_TEST=y
+CONFIG_KUNIT_EXAMPLE_TEST=y
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index da11bd62a4b82..3e51dc4febfdc 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -11,6 +11,7 @@ import argparse
import sys
import os
import time
+import shutil
from collections import namedtuple
from enum import Enum, auto
@@ -21,7 +22,7 @@ import kunit_parser
KunitResult = namedtuple('KunitResult', ['status','result'])
-KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 'build_dir'])
+KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 'build_dir', 'defconfig'])
class KunitStatus(Enum):
SUCCESS = auto()
@@ -29,8 +30,16 @@ class KunitStatus(Enum):
BUILD_FAILURE = auto()
TEST_FAILURE = auto()
+def create_default_kunitconfig():
+ if not os.path.exists(kunit_kernel.KUNITCONFIG_PATH):
+ shutil.copyfile('arch/um/configs/kunit_defconfig',
+ kunit_kernel.KUNITCONFIG_PATH)
+
def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
+ if request.defconfig:
+ create_default_kunitconfig()
+
config_start = time.time()
success = linux.build_reconfig(request.build_dir)
config_end = time.time()
@@ -99,13 +108,18 @@ def main(argv, linux):
'directory.',
type=str, default=None, metavar='build_dir')
+ run_parser.add_argument('--defconfig',
+ help='Uses a default kunitconfig.',
+ action='store_true')
+
cli_args = parser.parse_args(argv)
if cli_args.subcommand == 'run':
request = KunitRequest(cli_args.raw_output,
cli_args.timeout,
cli_args.jobs,
- cli_args.build_dir)
+ cli_args.build_dir,
+ cli_args.defconfig)
result = run_tests(linux, request)
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
index 07c0abf2f47df..bf38768353313 100644
--- a/tools/testing/kunit/kunit_kernel.py
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -14,6 +14,7 @@ import os
import kunit_config
KCONFIG_PATH = '.config'
+KUNITCONFIG_PATH = 'kunitconfig'
class ConfigError(Exception):
"""Represents an error trying to configure the Linux kernel."""
@@ -81,7 +82,7 @@ class LinuxSourceTree(object):
def __init__(self):
self._kconfig = kunit_config.Kconfig()
- self._kconfig.read_from_file('kunitconfig')
+ self._kconfig.read_from_file(KUNITCONFIG_PATH)
self._ops = LinuxSourceTreeOperations()
def clean(self):
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
* [PATCH v11 13/18] kunit: tool: add Python wrappers for running KUnit tests
From: Brendan Higgins @ 2019-07-17 1:55 UTC (permalink / raw)
To: frowand.list, gregkh, jpoimboe, keescook, kieran.bingham, mcgrof,
peterz, robh, sboyd, shuah, tytso, yamada.masahiro
Cc: devicetree, dri-devel, kunit-dev, linux-doc, linux-fsdevel,
linux-kbuild, linux-kernel, linux-kselftest, linux-nvdimm,
linux-um, Alexander.Levin, Tim.Bird, amir73il, dan.carpenter,
daniel, jdike, joel, julia.lawall, khilman, knut.omang, logang,
mpe, pmladek, rdunlap, richard, rientjes, rostedt, wfg, Felix Guo,
Brendan Higgins
In-Reply-To: <20190717015543.152251-1-brendanhiggins@google.com>
From: Felix Guo <felixguoxiuping@gmail.com>
The ultimate goal is to create minimal isolated test binaries; in the
meantime we are using UML to provide the infrastructure to run tests, so
define an abstract way to configure and run tests that allow us to
change the context in which tests are built without affecting the user.
This also makes pretty and dynamic error reporting, and a lot of other
nice features easier.
kunit_config.py:
- parse .config and Kconfig files.
kunit_kernel.py: provides helper functions to:
- configure the kernel using kunitconfig.
- build the kernel with the appropriate configuration.
- provide function to invoke the kernel and stream the output back.
kunit_parser.py: parses raw logs returned out by kunit_kernel and
displays them in a user friendly way.
test_data/*: samples of test data for testing kunit.py, kunit_config.py,
etc.
Signed-off-by: Felix Guo <felixguoxiuping@gmail.com>
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
---
tools/testing/kunit/.gitignore | 3 +
tools/testing/kunit/kunit.py | 116 +++++++
tools/testing/kunit/kunit_config.py | 66 ++++
tools/testing/kunit/kunit_kernel.py | 148 +++++++++
tools/testing/kunit/kunit_parser.py | 290 ++++++++++++++++++
tools/testing/kunit/kunit_tool_test.py | 206 +++++++++++++
.../test_is_test_passed-all_passed.log | 32 ++
.../test_data/test_is_test_passed-crash.log | 69 +++++
.../test_data/test_is_test_passed-failure.log | 36 +++
.../test_is_test_passed-no_tests_run.log | 75 +++++
.../test_output_isolated_correctly.log | 106 +++++++
.../test_data/test_read_from_file.kconfig | 17 +
12 files changed, 1164 insertions(+)
create mode 100644 tools/testing/kunit/.gitignore
create mode 100755 tools/testing/kunit/kunit.py
create mode 100644 tools/testing/kunit/kunit_config.py
create mode 100644 tools/testing/kunit/kunit_kernel.py
create mode 100644 tools/testing/kunit/kunit_parser.py
create mode 100755 tools/testing/kunit/kunit_tool_test.py
create mode 100644 tools/testing/kunit/test_data/test_is_test_passed-all_passed.log
create mode 100644 tools/testing/kunit/test_data/test_is_test_passed-crash.log
create mode 100644 tools/testing/kunit/test_data/test_is_test_passed-failure.log
create mode 100644 tools/testing/kunit/test_data/test_is_test_passed-no_tests_run.log
create mode 100644 tools/testing/kunit/test_data/test_output_isolated_correctly.log
create mode 100644 tools/testing/kunit/test_data/test_read_from_file.kconfig
diff --git a/tools/testing/kunit/.gitignore b/tools/testing/kunit/.gitignore
new file mode 100644
index 0000000000000..c791ff59a37a9
--- /dev/null
+++ b/tools/testing/kunit/.gitignore
@@ -0,0 +1,3 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
\ No newline at end of file
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
new file mode 100755
index 0000000000000..da11bd62a4b82
--- /dev/null
+++ b/tools/testing/kunit/kunit.py
@@ -0,0 +1,116 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# A thin wrapper on top of the KUnit Kernel
+#
+# Copyright (C) 2019, Google LLC.
+# Author: Felix Guo <felixguoxiuping@gmail.com>
+# Author: Brendan Higgins <brendanhiggins@google.com>
+
+import argparse
+import sys
+import os
+import time
+
+from collections import namedtuple
+from enum import Enum, auto
+
+import kunit_config
+import kunit_kernel
+import kunit_parser
+
+KunitResult = namedtuple('KunitResult', ['status','result'])
+
+KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 'build_dir'])
+
+class KunitStatus(Enum):
+ SUCCESS = auto()
+ CONFIG_FAILURE = auto()
+ BUILD_FAILURE = auto()
+ TEST_FAILURE = auto()
+
+def run_tests(linux: kunit_kernel.LinuxSourceTree,
+ request: KunitRequest) -> KunitResult:
+ config_start = time.time()
+ success = linux.build_reconfig(request.build_dir)
+ config_end = time.time()
+ if not success:
+ return KunitResult(KunitStatus.CONFIG_FAILURE, 'could not configure kernel')
+
+ kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
+
+ build_start = time.time()
+ success = linux.build_um_kernel(request.jobs, request.build_dir)
+ build_end = time.time()
+ if not success:
+ return KunitResult(KunitStatus.BUILD_FAILURE, 'could not build kernel')
+
+ kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
+ test_start = time.time()
+
+ test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,
+ [],
+ 'Tests not Parsed.')
+ if request.raw_output:
+ kunit_parser.raw_output(
+ linux.run_kernel(timeout=request.timeout))
+ else:
+ kunit_output = linux.run_kernel(timeout=request.timeout)
+ test_result = kunit_parser.parse_run_tests(kunit_output)
+ test_end = time.time()
+
+ kunit_parser.print_with_timestamp((
+ 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
+ 'building, %.3fs running\n') % (
+ test_end - config_start,
+ config_end - config_start,
+ build_end - build_start,
+ test_end - test_start))
+
+ if test_result.status != kunit_parser.TestStatus.SUCCESS:
+ return KunitResult(KunitStatus.TEST_FAILURE, test_result)
+ else:
+ return KunitResult(KunitStatus.SUCCESS, test_result)
+
+def main(argv, linux):
+ parser = argparse.ArgumentParser(
+ description='Helps writing and running KUnit tests.')
+ subparser = parser.add_subparsers(dest='subcommand')
+
+ run_parser = subparser.add_parser('run', help='Runs KUnit tests.')
+ run_parser.add_argument('--raw_output', help='don\'t format output from kernel',
+ action='store_true')
+
+ run_parser.add_argument('--timeout',
+ help='maximum number of seconds to allow for all tests '
+ 'to run. This does not include time taken to build the '
+ 'tests.',
+ type=int,
+ default=300,
+ metavar='timeout')
+
+ run_parser.add_argument('--jobs',
+ help='As in the make command, "Specifies the number of '
+ 'jobs (commands) to run simultaneously."',
+ type=int, default=8, metavar='jobs')
+
+ run_parser.add_argument('--build_dir',
+ help='As in the make command, it specifies the build '
+ 'directory.',
+ type=str, default=None, metavar='build_dir')
+
+ cli_args = parser.parse_args(argv)
+
+ if cli_args.subcommand == 'run':
+ request = KunitRequest(cli_args.raw_output,
+ cli_args.timeout,
+ cli_args.jobs,
+ cli_args.build_dir)
+ result = run_tests(linux, request)
+ if result.status != KunitStatus.SUCCESS:
+ sys.exit(1)
+ else:
+ parser.print_help()
+
+if __name__ == '__main__':
+ main(sys.argv[1:], kunit_kernel.LinuxSourceTree())
diff --git a/tools/testing/kunit/kunit_config.py b/tools/testing/kunit/kunit_config.py
new file mode 100644
index 0000000000000..ebf3942b23f51
--- /dev/null
+++ b/tools/testing/kunit/kunit_config.py
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Builds a .config from a kunitconfig.
+#
+# Copyright (C) 2019, Google LLC.
+# Author: Felix Guo <felixguoxiuping@gmail.com>
+# Author: Brendan Higgins <brendanhiggins@google.com>
+
+import collections
+import re
+
+CONFIG_IS_NOT_SET_PATTERN = r'^# CONFIG_\w+ is not set$'
+CONFIG_PATTERN = r'^CONFIG_\w+=\S+$'
+
+KconfigEntryBase = collections.namedtuple('KconfigEntry', ['raw_entry'])
+
+
+class KconfigEntry(KconfigEntryBase):
+
+ def __str__(self) -> str:
+ return self.raw_entry
+
+
+class KconfigParseError(Exception):
+ """Error parsing Kconfig defconfig or .config."""
+
+
+class Kconfig(object):
+ """Represents defconfig or .config specified using the Kconfig language."""
+
+ def __init__(self):
+ self._entries = []
+
+ def entries(self):
+ return set(self._entries)
+
+ def add_entry(self, entry: KconfigEntry) -> None:
+ self._entries.append(entry)
+
+ def is_subset_of(self, other: 'Kconfig') -> bool:
+ return self.entries().issubset(other.entries())
+
+ def write_to_file(self, path: str) -> None:
+ with open(path, 'w') as f:
+ for entry in self.entries():
+ f.write(str(entry) + '\n')
+
+ def parse_from_string(self, blob: str) -> None:
+ """Parses a string containing KconfigEntrys and populates this Kconfig."""
+ self._entries = []
+ is_not_set_matcher = re.compile(CONFIG_IS_NOT_SET_PATTERN)
+ config_matcher = re.compile(CONFIG_PATTERN)
+ for line in blob.split('\n'):
+ line = line.strip()
+ if not line:
+ continue
+ elif config_matcher.match(line) or is_not_set_matcher.match(line):
+ self._entries.append(KconfigEntry(line))
+ elif line[0] == '#':
+ continue
+ else:
+ raise KconfigParseError('Failed to parse: ' + line)
+
+ def read_from_file(self, path: str) -> None:
+ with open(path, 'r') as f:
+ self.parse_from_string(f.read())
diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
new file mode 100644
index 0000000000000..07c0abf2f47df
--- /dev/null
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Runs UML kernel, collects output, and handles errors.
+#
+# Copyright (C) 2019, Google LLC.
+# Author: Felix Guo <felixguoxiuping@gmail.com>
+# Author: Brendan Higgins <brendanhiggins@google.com>
+
+
+import logging
+import subprocess
+import os
+
+import kunit_config
+
+KCONFIG_PATH = '.config'
+
+class ConfigError(Exception):
+ """Represents an error trying to configure the Linux kernel."""
+
+
+class BuildError(Exception):
+ """Represents an error trying to build the Linux kernel."""
+
+
+class LinuxSourceTreeOperations(object):
+ """An abstraction over command line operations performed on a source tree."""
+
+ def make_mrproper(self):
+ try:
+ subprocess.check_output(['make', 'mrproper'])
+ except OSError as e:
+ raise ConfigError('Could not call make command: ' + e)
+ except subprocess.CalledProcessError as e:
+ raise ConfigError(e.output)
+
+ def make_olddefconfig(self, build_dir):
+ command = ['make', 'ARCH=um', 'olddefconfig']
+ if build_dir:
+ command += ['O=' + build_dir]
+ try:
+ subprocess.check_output(command)
+ except OSError as e:
+ raise ConfigError('Could not call make command: ' + e)
+ except subprocess.CalledProcessError as e:
+ raise ConfigError(e.output)
+
+ def make(self, jobs, build_dir):
+ command = ['make', 'ARCH=um', '--jobs=' + str(jobs)]
+ if build_dir:
+ command += ['O=' + build_dir]
+ try:
+ subprocess.check_output(command)
+ except OSError as e:
+ raise BuildError('Could not call execute make: ' + e)
+ except subprocess.CalledProcessError as e:
+ raise BuildError(e.output)
+
+ def linux_bin(self, params, timeout, build_dir):
+ """Runs the Linux UML binary. Must be named 'linux'."""
+ linux_bin = './linux'
+ if build_dir:
+ linux_bin = os.path.join(build_dir, 'linux')
+ process = subprocess.Popen(
+ [linux_bin] + params,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ process.wait(timeout=timeout)
+ return process
+
+
+def get_kconfig_path(build_dir):
+ kconfig_path = KCONFIG_PATH
+ if build_dir:
+ kconfig_path = os.path.join(build_dir, KCONFIG_PATH)
+ return kconfig_path
+
+class LinuxSourceTree(object):
+ """Represents a Linux kernel source tree with KUnit tests."""
+
+ def __init__(self):
+ self._kconfig = kunit_config.Kconfig()
+ self._kconfig.read_from_file('kunitconfig')
+ self._ops = LinuxSourceTreeOperations()
+
+ def clean(self):
+ try:
+ self._ops.make_mrproper()
+ except ConfigError as e:
+ logging.error(e)
+ return False
+ return True
+
+ def build_config(self, build_dir):
+ kconfig_path = get_kconfig_path(build_dir)
+ if build_dir and not os.path.exists(build_dir):
+ os.mkdir(build_dir)
+ self._kconfig.write_to_file(kconfig_path)
+ try:
+ self._ops.make_olddefconfig(build_dir)
+ except ConfigError as e:
+ logging.error(e)
+ return False
+ validated_kconfig = kunit_config.Kconfig()
+ validated_kconfig.read_from_file(kconfig_path)
+ if not self._kconfig.is_subset_of(validated_kconfig):
+ logging.error('Provided Kconfig is not contained in validated .config!')
+ return False
+ return True
+
+ def build_reconfig(self, build_dir):
+ """Creates a new .config if it is not a subset of the kunitconfig."""
+ kconfig_path = get_kconfig_path(build_dir)
+ if os.path.exists(kconfig_path):
+ existing_kconfig = kunit_config.Kconfig()
+ existing_kconfig.read_from_file(kconfig_path)
+ if not self._kconfig.is_subset_of(existing_kconfig):
+ print('Regenerating .config ...')
+ os.remove(kconfig_path)
+ return self.build_config(build_dir)
+ else:
+ return True
+ else:
+ print('Generating .config ...')
+ return self.build_config(build_dir)
+
+ def build_um_kernel(self, jobs, build_dir):
+ try:
+ self._ops.make_olddefconfig(build_dir)
+ self._ops.make(jobs, build_dir)
+ except (ConfigError, BuildError) as e:
+ logging.error(e)
+ return False
+ used_kconfig = kunit_config.Kconfig()
+ used_kconfig.read_from_file(get_kconfig_path(build_dir))
+ if not self._kconfig.is_subset_of(used_kconfig):
+ logging.error('Provided Kconfig is not contained in final config!')
+ return False
+ return True
+
+ def run_kernel(self, args=[], timeout=None, build_dir=None):
+ args.extend(['mem=256M'])
+ process = self._ops.linux_bin(args, timeout, build_dir)
+ with open('test.log', 'w') as f:
+ for line in process.stdout:
+ f.write(line.rstrip().decode('ascii') + '\n')
+ yield line.rstrip().decode('ascii')
diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py
new file mode 100644
index 0000000000000..f27f3d675c3bb
--- /dev/null
+++ b/tools/testing/kunit/kunit_parser.py
@@ -0,0 +1,290 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Parses test results from a kernel dmesg log.
+#
+# Copyright (C) 2019, Google LLC.
+# Author: Felix Guo <felixguoxiuping@gmail.com>
+# Author: Brendan Higgins <brendanhiggins@google.com>
+
+import re
+
+from collections import namedtuple
+from datetime import datetime
+from enum import Enum, auto
+from functools import reduce
+from typing import List
+
+TestResult = namedtuple('TestResult', ['status','suites','log'])
+
+class TestSuite(object):
+ def __init__(self):
+ self.status = None
+ self.name = None
+ self.cases = []
+
+ def __str__(self):
+ return 'TestSuite(' + self.status + ',' + self.name + ',' + str(self.cases) + ')'
+
+ def __repr__(self):
+ return str(self)
+
+class TestCase(object):
+ def __init__(self):
+ self.status = None
+ self.name = ''
+ self.log = []
+
+ def __str__(self):
+ return 'TestCase(' + self.status + ',' + self.name + ',' + str(self.log) + ')'
+
+ def __repr__(self):
+ return str(self)
+
+class TestStatus(Enum):
+ SUCCESS = auto()
+ FAILURE = auto()
+ TEST_CRASHED = auto()
+ NO_TESTS = auto()
+
+kunit_start_re = re.compile(r'^TAP version [0-9]+$')
+kunit_end_re = re.compile('List of all partitions:')
+
+def isolate_kunit_output(kernel_output):
+ started = False
+ for line in kernel_output:
+ if kunit_start_re.match(line):
+ started = True
+ yield line
+ elif kunit_end_re.match(line):
+ break
+ elif started:
+ yield line
+
+def raw_output(kernel_output):
+ for line in kernel_output:
+ print(line)
+
+DIVIDER = '=' * 60
+
+RESET = '\033[0;0m'
+
+def red(text):
+ return '\033[1;31m' + text + RESET
+
+def yellow(text):
+ return '\033[1;33m' + text + RESET
+
+def green(text):
+ return '\033[1;32m' + text + RESET
+
+def print_with_timestamp(message):
+ print('[%s] %s' % (datetime.now().strftime('%H:%M:%S'), message))
+
+def format_suite_divider(message):
+ return '======== ' + message + ' ========'
+
+def print_suite_divider(message):
+ print_with_timestamp(DIVIDER)
+ print_with_timestamp(format_suite_divider(message))
+
+def print_log(log):
+ for m in log:
+ print_with_timestamp(m)
+
+TAP_ENTRIES = re.compile(r'^(TAP|\t?ok|\t?not ok|\t?[0-9]+\.\.[0-9]+|\t?#).*$')
+
+def consume_non_diagnositic(lines: List[str]) -> None:
+ while not TAP_ENTRIES.match(lines[0]):
+ lines.pop(0)
+
+def save_non_diagnositic(lines: List[str], test_case: TestCase) -> None:
+ while not TAP_ENTRIES.match(lines[0]):
+ test_case.log.append(lines[0])
+ lines.pop(0)
+
+OkNotOkResult = namedtuple('OkNotOkResult', ['is_ok','description', 'text'])
+
+OK_NOT_OK_SUBTEST = re.compile(r'^\t(ok|not ok) [0-9]+ - (.*)$')
+
+OK_NOT_OK_MODULE = re.compile(r'^(ok|not ok) [0-9]+ - (.*)$')
+
+def parse_ok_not_ok_test_case(lines: List[str], test_case: TestCase) -> bool:
+ save_non_diagnositic(lines, test_case)
+ line = lines[0]
+ match = OK_NOT_OK_SUBTEST.match(line)
+ if match:
+ test_case.log.append(lines.pop(0))
+ test_case.name = match.group(2)
+ if test_case.status == TestStatus.TEST_CRASHED:
+ return True
+ if match.group(1) == 'ok':
+ test_case.status = TestStatus.SUCCESS
+ else:
+ test_case.status = TestStatus.FAILURE
+ return True
+ else:
+ return False
+
+SUBTEST_DIAGNOSTIC = re.compile(r'^\t# .*?: (.*)$')
+DIAGNOSTIC_CRASH_MESSAGE = 'kunit test case crashed!'
+
+def parse_diagnostic(lines: List[str], test_case: TestCase) -> bool:
+ save_non_diagnositic(lines, test_case)
+ line = lines[0]
+ match = SUBTEST_DIAGNOSTIC.match(line)
+ if match:
+ test_case.log.append(lines.pop(0))
+ if match.group(1) == DIAGNOSTIC_CRASH_MESSAGE:
+ test_case.status = TestStatus.TEST_CRASHED
+ return True
+ else:
+ return False
+
+def parse_test_case(lines: List[str]) -> TestCase:
+ test_case = TestCase()
+ save_non_diagnositic(lines, test_case)
+ while parse_diagnostic(lines, test_case):
+ pass
+ if parse_ok_not_ok_test_case(lines, test_case):
+ return test_case
+ else:
+ return None
+
+SUBTEST_HEADER = re.compile(r'^\t# Subtest: (.*)$')
+
+def parse_subtest_header(lines: List[str]) -> str:
+ consume_non_diagnositic(lines)
+ match = SUBTEST_HEADER.match(lines[0])
+ if match:
+ lines.pop(0)
+ return match.group(1)
+ else:
+ return None
+
+SUBTEST_PLAN = re.compile(r'\t[0-9]+\.\.([0-9]+)')
+
+def parse_subtest_plan(lines: List[str]) -> int:
+ consume_non_diagnositic(lines)
+ match = SUBTEST_PLAN.match(lines[0])
+ if match:
+ lines.pop(0)
+ return match.group(1)
+ else:
+ return None
+
+def max_status(left: TestStatus, right: TestStatus) -> TestStatus:
+ if left == TestStatus.TEST_CRASHED or right == TestStatus.TEST_CRASHED:
+ return TestStatus.TEST_CRASHED
+ elif left == TestStatus.FAILURE or right == TestStatus.FAILURE:
+ return TestStatus.FAILURE
+ elif left != TestStatus.SUCCESS:
+ return left
+ elif right != TestStatus.SUCCESS:
+ return right
+ else:
+ return TestStatus.SUCCESS
+
+def parse_ok_not_ok_test_suite(lines: List[str], test_suite: TestSuite) -> bool:
+ consume_non_diagnositic(lines)
+ line = lines[0]
+ match = OK_NOT_OK_MODULE.match(line)
+ if match:
+ lines.pop(0)
+ if match.group(1) == 'ok':
+ test_suite.status = TestStatus.SUCCESS
+ else:
+ test_suite.status = TestStatus.FAILURE
+ return True
+ else:
+ return False
+
+def bubble_up_errors(to_status, status_container_list) -> TestStatus:
+ status_list = map(to_status, status_container_list)
+ return reduce(max_status, status_list, TestStatus.SUCCESS)
+
+def bubble_up_test_case_errors(test_suite: TestSuite) -> TestStatus:
+ max_test_case_status = bubble_up_errors(lambda x: x.status, test_suite.cases)
+ return max_status(max_test_case_status, test_suite.status)
+
+def parse_test_suite(lines: List[str]) -> TestSuite:
+ if not lines:
+ return None
+ consume_non_diagnositic(lines)
+ test_suite = TestSuite()
+ test_suite.status = TestStatus.SUCCESS
+ name = parse_subtest_header(lines)
+ if not name:
+ return None
+ test_suite.name = name
+ test_case_num = parse_subtest_plan(lines)
+ if not test_case_num:
+ return None
+ test_case = parse_test_case(lines)
+ while test_case:
+ test_suite.cases.append(test_case)
+ test_case = parse_test_case(lines)
+ if parse_ok_not_ok_test_suite(lines, test_suite):
+ test_suite.status = bubble_up_test_case_errors(test_suite)
+ return test_suite
+ else:
+ print('failed to parse end of suite' + lines[0])
+ return None
+
+TAP_HEADER = re.compile(r'^TAP version 14$')
+
+def parse_tap_header(lines: List[str]) -> bool:
+ consume_non_diagnositic(lines)
+ if TAP_HEADER.match(lines[0]):
+ lines.pop(0)
+ return True
+ else:
+ return False
+
+def bubble_up_suite_errors(test_suite_list: List[TestSuite]) -> TestStatus:
+ return bubble_up_errors(lambda x: x.status, test_suite_list)
+
+def parse_test_result(lines: List[str]) -> TestResult:
+ if not lines:
+ return TestResult(TestStatus.NO_TESTS, [], lines)
+ consume_non_diagnositic(lines)
+ if not parse_tap_header(lines):
+ return None
+ test_suites = []
+ test_suite = parse_test_suite(lines)
+ while test_suite:
+ test_suites.append(test_suite)
+ test_suite = parse_test_suite(lines)
+ return TestResult(bubble_up_suite_errors(test_suites), test_suites, lines)
+
+def parse_run_tests(kernel_output) -> TestResult:
+ total_tests = 0
+ failed_tests = 0
+ crashed_tests = 0
+ test_result = parse_test_result(list(isolate_kunit_output(kernel_output)))
+ for test_suite in test_result.suites:
+ if test_suite.status == TestStatus.SUCCESS:
+ print_suite_divider(green('[PASSED] ') + test_suite.name)
+ elif test_suite.status == TestStatus.TEST_CRASHED:
+ print_suite_divider(red('[CRASHED] ' + test_suite.name))
+ else:
+ print_suite_divider(red('[FAILED] ') + test_suite.name)
+ for test_case in test_suite.cases:
+ total_tests += 1
+ if test_case.status == TestStatus.SUCCESS:
+ print_with_timestamp(green('[PASSED] ') + test_case.name)
+ elif test_case.status == TestStatus.TEST_CRASHED:
+ crashed_tests += 1
+ print_with_timestamp(red('[CRASHED] ' + test_case.name))
+ print_log(map(yellow, test_case.log))
+ print_with_timestamp('')
+ else:
+ failed_tests += 1
+ print_with_timestamp(red('[FAILED] ') + test_case.name)
+ print_log(map(yellow, test_case.log))
+ print_with_timestamp('')
+ print_with_timestamp(DIVIDER)
+ fmt = green if test_result.status == TestStatus.SUCCESS else red
+ print_with_timestamp(
+ fmt('Testing complete. %d tests run. %d failed. %d crashed.' %
+ (total_tests, failed_tests, crashed_tests)))
+ return test_result
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
new file mode 100755
index 0000000000000..4a12baa0cd4e0
--- /dev/null
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -0,0 +1,206 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# A collection of tests for tools/testing/kunit/kunit.py
+#
+# Copyright (C) 2019, Google LLC.
+# Author: Brendan Higgins <brendanhiggins@google.com>
+
+import unittest
+from unittest import mock
+
+import tempfile, shutil # Handling test_tmpdir
+
+import os
+
+import kunit_config
+import kunit_parser
+import kunit_kernel
+import kunit
+
+test_tmpdir = ''
+
+def setUpModule():
+ global test_tmpdir
+ test_tmpdir = tempfile.mkdtemp()
+
+def tearDownModule():
+ shutil.rmtree(test_tmpdir)
+
+def get_absolute_path(path):
+ return os.path.join(os.path.dirname(__file__), path)
+
+class KconfigTest(unittest.TestCase):
+
+ def test_is_subset_of(self):
+ kconfig0 = kunit_config.Kconfig()
+ self.assertTrue(kconfig0.is_subset_of(kconfig0))
+
+ kconfig1 = kunit_config.Kconfig()
+ kconfig1.add_entry(kunit_config.KconfigEntry('CONFIG_TEST=y'))
+ self.assertTrue(kconfig1.is_subset_of(kconfig1))
+ self.assertTrue(kconfig0.is_subset_of(kconfig1))
+ self.assertFalse(kconfig1.is_subset_of(kconfig0))
+
+ def test_read_from_file(self):
+ kconfig = kunit_config.Kconfig()
+ kconfig_path = get_absolute_path(
+ 'test_data/test_read_from_file.kconfig')
+
+ kconfig.read_from_file(kconfig_path)
+
+ expected_kconfig = kunit_config.Kconfig()
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_UML=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_MMU=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_TEST=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_EXAMPLE_TEST=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('# CONFIG_MK8 is not set'))
+
+ self.assertEqual(kconfig.entries(), expected_kconfig.entries())
+
+ def test_write_to_file(self):
+ kconfig_path = os.path.join(test_tmpdir, '.config')
+
+ expected_kconfig = kunit_config.Kconfig()
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_UML=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_MMU=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_TEST=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('CONFIG_EXAMPLE_TEST=y'))
+ expected_kconfig.add_entry(
+ kunit_config.KconfigEntry('# CONFIG_MK8 is not set'))
+
+ expected_kconfig.write_to_file(kconfig_path)
+
+ actual_kconfig = kunit_config.Kconfig()
+ actual_kconfig.read_from_file(kconfig_path)
+
+ self.assertEqual(actual_kconfig.entries(),
+ expected_kconfig.entries())
+
+class KUnitParserTest(unittest.TestCase):
+
+ def assertContains(self, needle, haystack):
+ for line in haystack:
+ if needle in line:
+ return
+ raise AssertionError('"' +
+ str(needle) + '" not found in "' + str(haystack) + '"!')
+
+ def test_output_isolated_correctly(self):
+ log_path = get_absolute_path(
+ 'test_data/test_output_isolated_correctly.log')
+ file = open(log_path)
+ result = kunit_parser.isolate_kunit_output(file.readlines())
+ self.assertContains('TAP version 14\n', result)
+ self.assertContains(' # Subtest: example', result)
+ self.assertContains(' 1..2', result)
+ self.assertContains(' ok 1 - example_simple_test', result)
+ self.assertContains(' ok 2 - example_mock_test', result)
+ self.assertContains('ok 1 - example', result)
+ file.close()
+
+ def test_parse_successful_test_log(self):
+ all_passed_log = get_absolute_path(
+ 'test_data/test_is_test_passed-all_passed.log')
+ file = open(all_passed_log)
+ result = kunit_parser.parse_run_tests(file.readlines())
+ self.assertEqual(
+ kunit_parser.TestStatus.SUCCESS,
+ result.status)
+ file.close()
+
+ def test_parse_failed_test_log(self):
+ failed_log = get_absolute_path(
+ 'test_data/test_is_test_passed-failure.log')
+ file = open(failed_log)
+ result = kunit_parser.parse_run_tests(file.readlines())
+ self.assertEqual(
+ kunit_parser.TestStatus.FAILURE,
+ result.status)
+ file.close()
+
+ def test_no_tests(self):
+ empty_log = get_absolute_path(
+ 'test_data/test_is_test_passed-no_tests_run.log')
+ file = open(empty_log)
+ result = kunit_parser.parse_run_tests(
+ kunit_parser.isolate_kunit_output(file.readlines()))
+ self.assertEqual(0, len(result.suites))
+ self.assertEqual(
+ kunit_parser.TestStatus.NO_TESTS,
+ result.status)
+ file.close()
+
+ def test_crashed_test(self):
+ crashed_log = get_absolute_path(
+ 'test_data/test_is_test_passed-crash.log')
+ file = open(crashed_log)
+ result = kunit_parser.parse_run_tests(file.readlines())
+ self.assertEqual(
+ kunit_parser.TestStatus.TEST_CRASHED,
+ result.status)
+ file.close()
+
+class StrContains(str):
+ def __eq__(self, other):
+ return self in other
+
+class KUnitMainTest(unittest.TestCase):
+ def setUp(self):
+ path = get_absolute_path('test_data/test_is_test_passed-all_passed.log')
+ file = open(path)
+ all_passed_log = file.readlines()
+ self.print_patch = mock.patch('builtins.print')
+ self.print_mock = self.print_patch.start()
+ self.linux_source_mock = mock.Mock()
+ self.linux_source_mock.build_reconfig = mock.Mock(return_value=True)
+ self.linux_source_mock.build_um_kernel = mock.Mock(return_value=True)
+ self.linux_source_mock.run_kernel = mock.Mock(return_value=all_passed_log)
+
+ def tearDown(self):
+ self.print_patch.stop()
+ pass
+
+ def test_run_passes_args_pass(self):
+ kunit.main(['run'], self.linux_source_mock)
+ assert self.linux_source_mock.build_reconfig.call_count == 1
+ assert self.linux_source_mock.run_kernel.call_count == 1
+ self.print_mock.assert_any_call(StrContains('Testing complete.'))
+
+ def test_run_passes_args_fail(self):
+ self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
+ with self.assertRaises(SystemExit) as e:
+ kunit.main(['run'], self.linux_source_mock)
+ assert type(e.exception) == SystemExit
+ assert e.exception.code == 1
+ assert self.linux_source_mock.build_reconfig.call_count == 1
+ assert self.linux_source_mock.run_kernel.call_count == 1
+ self.print_mock.assert_any_call(StrContains(' 0 tests run'))
+
+ def test_run_raw_output(self):
+ self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
+ kunit.main(['run', '--raw_output'], self.linux_source_mock)
+ assert self.linux_source_mock.build_reconfig.call_count == 1
+ assert self.linux_source_mock.run_kernel.call_count == 1
+ for kall in self.print_mock.call_args_list:
+ assert kall != mock.call(StrContains('Testing complete.'))
+ assert kall != mock.call(StrContains(' 0 tests run'))
+
+ def test_run_timeout(self):
+ timeout = 3453
+ kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock)
+ assert self.linux_source_mock.build_reconfig.call_count == 1
+ self.linux_source_mock.run_kernel.assert_called_once_with(timeout=timeout)
+ self.print_mock.assert_any_call(StrContains('Testing complete.'))
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/testing/kunit/test_data/test_is_test_passed-all_passed.log b/tools/testing/kunit/test_data/test_is_test_passed-all_passed.log
new file mode 100644
index 0000000000000..62ebc0288355c
--- /dev/null
+++ b/tools/testing/kunit/test_data/test_is_test_passed-all_passed.log
@@ -0,0 +1,32 @@
+TAP version 14
+ # Subtest: sysctl_test
+ 1..8
+ # sysctl_test_dointvec_null_tbl_data: sysctl_test_dointvec_null_tbl_data passed
+ ok 1 - sysctl_test_dointvec_null_tbl_data
+ # sysctl_test_dointvec_table_maxlen_unset: sysctl_test_dointvec_table_maxlen_unset passed
+ ok 2 - sysctl_test_dointvec_table_maxlen_unset
+ # sysctl_test_dointvec_table_len_is_zero: sysctl_test_dointvec_table_len_is_zero passed
+ ok 3 - sysctl_test_dointvec_table_len_is_zero
+ # sysctl_test_dointvec_table_read_but_position_set: sysctl_test_dointvec_table_read_but_position_set passed
+ ok 4 - sysctl_test_dointvec_table_read_but_position_set
+ # sysctl_test_dointvec_happy_single_positive: sysctl_test_dointvec_happy_single_positive passed
+ ok 5 - sysctl_test_dointvec_happy_single_positive
+ # sysctl_test_dointvec_happy_single_negative: sysctl_test_dointvec_happy_single_negative passed
+ ok 6 - sysctl_test_dointvec_happy_single_negative
+ # sysctl_test_dointvec_single_less_int_min: sysctl_test_dointvec_single_less_int_min passed
+ ok 7 - sysctl_test_dointvec_single_less_int_min
+ # sysctl_test_dointvec_single_greater_int_max: sysctl_test_dointvec_single_greater_int_max passed
+ ok 8 - sysctl_test_dointvec_single_greater_int_max
+kunit sysctl_test: all tests passed
+ok 1 - sysctl_test
+ # Subtest: example
+ 1..2
+init_suite
+ # example_simple_test: initializing
+ # example_simple_test: example_simple_test passed
+ ok 1 - example_simple_test
+ # example_mock_test: initializing
+ # example_mock_test: example_mock_test passed
+ ok 2 - example_mock_test
+kunit example: all tests passed
+ok 2 - example
diff --git a/tools/testing/kunit/test_data/test_is_test_passed-crash.log b/tools/testing/kunit/test_data/test_is_test_passed-crash.log
new file mode 100644
index 0000000000000..0b249870c8be4
--- /dev/null
+++ b/tools/testing/kunit/test_data/test_is_test_passed-crash.log
@@ -0,0 +1,69 @@
+printk: console [tty0] enabled
+printk: console [mc-1] enabled
+TAP version 14
+ # Subtest: sysctl_test
+ 1..8
+ # sysctl_test_dointvec_null_tbl_data: sysctl_test_dointvec_null_tbl_data passed
+ ok 1 - sysctl_test_dointvec_null_tbl_data
+ # sysctl_test_dointvec_table_maxlen_unset: sysctl_test_dointvec_table_maxlen_unset passed
+ ok 2 - sysctl_test_dointvec_table_maxlen_unset
+ # sysctl_test_dointvec_table_len_is_zero: sysctl_test_dointvec_table_len_is_zero passed
+ ok 3 - sysctl_test_dointvec_table_len_is_zero
+ # sysctl_test_dointvec_table_read_but_position_set: sysctl_test_dointvec_table_read_but_position_set passed
+ ok 4 - sysctl_test_dointvec_table_read_but_position_set
+ # sysctl_test_dointvec_happy_single_positive: sysctl_test_dointvec_happy_single_positive passed
+ ok 5 - sysctl_test_dointvec_happy_single_positive
+ # sysctl_test_dointvec_happy_single_negative: sysctl_test_dointvec_happy_single_negative passed
+ ok 6 - sysctl_test_dointvec_happy_single_negative
+ # sysctl_test_dointvec_single_less_int_min: sysctl_test_dointvec_single_less_int_min passed
+ ok 7 - sysctl_test_dointvec_single_less_int_min
+ # sysctl_test_dointvec_single_greater_int_max: sysctl_test_dointvec_single_greater_int_max passed
+ ok 8 - sysctl_test_dointvec_single_greater_int_max
+kunit sysctl_test: all tests passed
+ok 1 - sysctl_test
+ # Subtest: example
+ 1..2
+init_suite
+ # example_simple_test: initializing
+Stack:
+ 6016f7db 6f81bd30 6f81bdd0 60021450
+ 6024b0e8 60021440 60018bbe 16f81bdc0
+ 00000001 6f81bd30 6f81bd20 6f81bdd0
+Call Trace:
+ [<6016f7db>] ? kunit_try_run_case+0xab/0xf0
+ [<60021450>] ? set_signals+0x0/0x60
+ [<60021440>] ? get_signals+0x0/0x10
+ [<60018bbe>] ? kunit_um_run_try_catch+0x5e/0xc0
+ [<60021450>] ? set_signals+0x0/0x60
+ [<60021440>] ? get_signals+0x0/0x10
+ [<60018bb3>] ? kunit_um_run_try_catch+0x53/0xc0
+ [<6016f321>] ? kunit_run_case_catch_errors+0x121/0x1a0
+ [<60018b60>] ? kunit_um_run_try_catch+0x0/0xc0
+ [<600189e0>] ? kunit_um_throw+0x0/0x180
+ [<6016f730>] ? kunit_try_run_case+0x0/0xf0
+ [<6016f600>] ? kunit_catch_run_case+0x0/0x130
+ [<6016edd0>] ? kunit_vprintk+0x0/0x30
+ [<6016ece0>] ? kunit_fail+0x0/0x40
+ [<6016eca0>] ? kunit_abort+0x0/0x40
+ [<6016ed20>] ? kunit_printk_emit+0x0/0xb0
+ [<6016f200>] ? kunit_run_case_catch_errors+0x0/0x1a0
+ [<6016f46e>] ? kunit_run_tests+0xce/0x260
+ [<6005b390>] ? unregister_console+0x0/0x190
+ [<60175b70>] ? suite_kunit_initexample_test_suite+0x0/0x20
+ [<60001cbb>] ? do_one_initcall+0x0/0x197
+ [<60001d47>] ? do_one_initcall+0x8c/0x197
+ [<6005cd20>] ? irq_to_desc+0x0/0x30
+ [<60002005>] ? kernel_init_freeable+0x1b3/0x272
+ [<6005c5ec>] ? printk+0x0/0x9b
+ [<601c0086>] ? kernel_init+0x26/0x160
+ [<60014442>] ? new_thread_handler+0x82/0xc0
+
+ # example_simple_test: kunit test case crashed!
+ # example_simple_test: example_simple_test failed
+ not ok 1 - example_simple_test
+ # example_mock_test: initializing
+ # example_mock_test: example_mock_test passed
+ ok 2 - example_mock_test
+kunit example: one or more tests failed
+not ok 2 - example
+List of all partitions:
diff --git a/tools/testing/kunit/test_data/test_is_test_passed-failure.log b/tools/testing/kunit/test_data/test_is_test_passed-failure.log
new file mode 100644
index 0000000000000..2ed870b6de72f
--- /dev/null
+++ b/tools/testing/kunit/test_data/test_is_test_passed-failure.log
@@ -0,0 +1,36 @@
+TAP version 14
+ # Subtest: sysctl_test
+ 1..8
+ # sysctl_test_dointvec_null_tbl_data: sysctl_test_dointvec_null_tbl_data passed
+ ok 1 - sysctl_test_dointvec_null_tbl_data
+ # sysctl_test_dointvec_table_maxlen_unset: sysctl_test_dointvec_table_maxlen_unset passed
+ ok 2 - sysctl_test_dointvec_table_maxlen_unset
+ # sysctl_test_dointvec_table_len_is_zero: sysctl_test_dointvec_table_len_is_zero passed
+ ok 3 - sysctl_test_dointvec_table_len_is_zero
+ # sysctl_test_dointvec_table_read_but_position_set: sysctl_test_dointvec_table_read_but_position_set passed
+ ok 4 - sysctl_test_dointvec_table_read_but_position_set
+ # sysctl_test_dointvec_happy_single_positive: sysctl_test_dointvec_happy_single_positive passed
+ ok 5 - sysctl_test_dointvec_happy_single_positive
+ # sysctl_test_dointvec_happy_single_negative: sysctl_test_dointvec_happy_single_negative passed
+ ok 6 - sysctl_test_dointvec_happy_single_negative
+ # sysctl_test_dointvec_single_less_int_min: sysctl_test_dointvec_single_less_int_min passed
+ ok 7 - sysctl_test_dointvec_single_less_int_min
+ # sysctl_test_dointvec_single_greater_int_max: sysctl_test_dointvec_single_greater_int_max passed
+ ok 8 - sysctl_test_dointvec_single_greater_int_max
+kunit sysctl_test: all tests passed
+ok 1 - sysctl_test
+ # Subtest: example
+ 1..2
+init_suite
+ # example_simple_test: initializing
+ # example_simple_test: EXPECTATION FAILED at kunit/example-test.c:30
+ Expected 1 + 1 == 3, but
+ 1 + 1 == 2
+ 3 == 3
+ # example_simple_test: example_simple_test failed
+ not ok 1 - example_simple_test
+ # example_mock_test: initializing
+ # example_mock_test: example_mock_test passed
+ ok 2 - example_mock_test
+kunit example: one or more tests failed
+not ok 2 - example
diff --git a/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run.log b/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run.log
new file mode 100644
index 0000000000000..ba69f5c94b75f
--- /dev/null
+++ b/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run.log
@@ -0,0 +1,75 @@
+Core dump limits :
+ soft - 0
+ hard - NONE
+Checking environment variables for a tempdir...none found
+Checking if /dev/shm is on tmpfs...OK
+Checking PROT_EXEC mmap in /dev/shm...OK
+Adding 24743936 bytes to physical memory to account for exec-shield gap
+Linux version 4.12.0-rc3-00010-g7319eb35f493-dirty (brendanhiggins@mactruck.svl.corp.google.com) (gcc version 7.3.0 (Debian 7.3.0-5) ) #29 Thu Mar 15 14:57:19 PDT 2018
+Built 1 zonelists in Zone order, mobility grouping on. Total pages: 14038
+Kernel command line: root=98:0
+PID hash table entries: 256 (order: -1, 2048 bytes)
+Dentry cache hash table entries: 8192 (order: 4, 65536 bytes)
+Inode-cache hash table entries: 4096 (order: 3, 32768 bytes)
+Memory: 27868K/56932K available (1681K kernel code, 480K rwdata, 400K rodata, 89K init, 205K bss, 29064K reserved, 0K cma-reserved)
+SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1
+NR_IRQS:15
+clocksource: timer: mask: 0xffffffffffffffff max_cycles: 0x1cd42e205, max_idle_ns: 881590404426 ns
+Calibrating delay loop... 7384.26 BogoMIPS (lpj=36921344)
+pid_max: default: 32768 minimum: 301
+Mount-cache hash table entries: 512 (order: 0, 4096 bytes)
+Mountpoint-cache hash table entries: 512 (order: 0, 4096 bytes)
+Checking that host ptys support output SIGIO...Yes
+Checking that host ptys support SIGIO on close...No, enabling workaround
+Using 2.6 host AIO
+clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 19112604462750000 ns
+futex hash table entries: 256 (order: 0, 6144 bytes)
+clocksource: Switched to clocksource timer
+console [stderr0] disabled
+mconsole (version 2) initialized on /usr/local/google/home/brendanhiggins/.uml/6Ijecl/mconsole
+Checking host MADV_REMOVE support...OK
+workingset: timestamp_bits=62 max_order=13 bucket_order=0
+Block layer SCSI generic (bsg) driver version 0.4 loaded (major 254)
+io scheduler noop registered
+io scheduler deadline registered
+io scheduler cfq registered (default)
+io scheduler mq-deadline registered
+io scheduler kyber registered
+Initialized stdio console driver
+Using a channel type which is configured out of UML
+setup_one_line failed for device 1 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 2 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 3 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 4 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 5 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 6 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 7 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 8 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 9 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 10 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 11 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 12 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 13 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 14 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 15 : Configuration failed
+Console initialized on /dev/tty0
+console [tty0] enabled
+console [mc-1] enabled
+List of all partitions:
+No filesystem could mount root, tried:
+
+Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0)
diff --git a/tools/testing/kunit/test_data/test_output_isolated_correctly.log b/tools/testing/kunit/test_data/test_output_isolated_correctly.log
new file mode 100644
index 0000000000000..94a6b3aeaa922
--- /dev/null
+++ b/tools/testing/kunit/test_data/test_output_isolated_correctly.log
@@ -0,0 +1,106 @@
+Linux version 5.1.0-rc7-00061-g04652f1cb4aa0 (brendanhiggins@mactruck.svl.corp.google.com) (gcc version 7.3.0 (Debian 7.3.0-18)) #163 Wed May 8 16:18:20 PDT 2019
+Built 1 zonelists, mobility grouping on. Total pages: 69906
+Kernel command line: mem=256M root=98:0
+Dentry cache hash table entries: 65536 (order: 7, 524288 bytes)
+Inode-cache hash table entries: 32768 (order: 6, 262144 bytes)
+Memory: 254468K/283500K available (1734K kernel code, 489K rwdata, 396K rodata, 85K init, 216K bss, 29032K reserved, 0K cma-reserved)
+SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1
+NR_IRQS: 15
+clocksource: timer: mask: 0xffffffffffffffff max_cycles: 0x1cd42e205, max_idle_ns: 881590404426 ns
+------------[ cut here ]------------
+WARNING: CPU: 0 PID: 0 at kernel/time/clockevents.c:458 clockevents_register_device+0x143/0x160
+posix-timer cpumask == cpu_all_mask, using cpu_possible_mask instead
+CPU: 0 PID: 0 Comm: swapper Not tainted 5.1.0-rc7-00061-g04652f1cb4aa0 #163
+Stack:
+ 6005cc00 60233e18 60233e60 60233e18
+ 60233e60 00000009 00000000 6002a1b4
+ 1ca00000000 60071c23 60233e78 100000000000062
+Call Trace:
+ [<600214c5>] ? os_is_signal_stack+0x15/0x30
+ [<6005c5ec>] ? printk+0x0/0x9b
+ [<6001597e>] ? show_stack+0xbe/0x1c0
+ [<6005cc00>] ? __printk_safe_exit+0x0/0x40
+ [<6002a1b4>] ? __warn+0x144/0x170
+ [<60071c23>] ? clockevents_register_device+0x143/0x160
+ [<60021440>] ? get_signals+0x0/0x10
+ [<6005c5ec>] ? printk+0x0/0x9b
+ [<6002a27b>] ? warn_slowpath_fmt+0x9b/0xb0
+ [<6005c5ec>] ? printk+0x0/0x9b
+ [<6002a1e0>] ? warn_slowpath_fmt+0x0/0xb0
+ [<6005c5ec>] ? printk+0x0/0x9b
+ [<60021440>] ? get_signals+0x0/0x10
+ [<600213f0>] ? block_signals+0x0/0x20
+ [<60071c23>] ? clockevents_register_device+0x143/0x160
+ [<60021440>] ? get_signals+0x0/0x10
+ [<600213f0>] ? block_signals+0x0/0x20
+ [<6005c5ec>] ? printk+0x0/0x9b
+ [<60001bc8>] ? start_kernel+0x477/0x56a
+ [<600036f1>] ? start_kernel_proc+0x46/0x4d
+ [<60014442>] ? new_thread_handler+0x82/0xc0
+
+random: get_random_bytes called from print_oops_end_marker+0x4c/0x60 with crng_init=0
+---[ end trace c83434852b3702d3 ]---
+Calibrating delay loop... 6958.28 BogoMIPS (lpj=34791424)
+pid_max: default: 32768 minimum: 301
+Mount-cache hash table entries: 1024 (order: 1, 8192 bytes)
+Mountpoint-cache hash table entries: 1024 (order: 1, 8192 bytes)
+*** VALIDATE proc ***
+Checking that host ptys support output SIGIO...Yes
+Checking that host ptys support SIGIO on close...No, enabling workaround
+clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 19112604462750000 ns
+futex hash table entries: 256 (order: 0, 6144 bytes)
+clocksource: Switched to clocksource timer
+printk: console [stderr0] disabled
+mconsole (version 2) initialized on /usr/local/google/home/brendanhiggins/.uml/VZ2qMm/mconsole
+Checking host MADV_REMOVE support...OK
+workingset: timestamp_bits=62 max_order=16 bucket_order=0
+Block layer SCSI generic (bsg) driver version 0.4 loaded (major 254)
+io scheduler mq-deadline registered
+io scheduler kyber registered
+Initialized stdio console driver
+Using a channel type which is configured out of UML
+setup_one_line failed for device 1 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 2 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 3 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 4 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 5 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 6 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 7 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 8 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 9 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 10 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 11 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 12 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 13 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 14 : Configuration failed
+Using a channel type which is configured out of UML
+setup_one_line failed for device 15 : Configuration failed
+Console initialized on /dev/tty0
+printk: console [tty0] enabled
+printk: console [mc-1] enabled
+TAP version 14
+ # Subtest: example
+ 1..2
+init_suite
+ # example_simple_test: initializing
+ # example_simple_test: example_simple_test passed
+ ok 1 - example_simple_test
+ # example_mock_test: initializing
+ # example_mock_test: example_mock_test passed
+ ok 2 - example_mock_test
+kunit example: all tests passed
+ok 1 - example
+List of all partitions:
diff --git a/tools/testing/kunit/test_data/test_read_from_file.kconfig b/tools/testing/kunit/test_data/test_read_from_file.kconfig
new file mode 100644
index 0000000000000..d2a4928ac773b
--- /dev/null
+++ b/tools/testing/kunit/test_data/test_read_from_file.kconfig
@@ -0,0 +1,17 @@
+#
+# Automatically generated file; DO NOT EDIT.
+# User Mode Linux/x86 4.12.0-rc3 Kernel Configuration
+#
+CONFIG_UML=y
+CONFIG_MMU=y
+
+#
+# UML-specific options
+#
+
+#
+# Host processor type and features
+#
+# CONFIG_MK8 is not set
+CONFIG_TEST=y
+CONFIG_EXAMPLE_TEST=y
--
2.22.0.510.g264f2c817a-goog
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox