[PATCH v2 08/13] mm: update shmem_[kernel]_file_*() functions to use vma_flags_t

Lorenzo Stoakes
2026-01-22T16:08:28Z

In order to be able to use only vma_flags_t in vm_area_desc we must adjust
shmem file setup functions to operate in terms of vma_flags_t rather than
vm_flags_t.

This patch makes this change and updates all callers to use the new
functions.

No functional changes intended.

Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
---
arch/x86/kernel/cpu/sgx/ioctl.c | 2 +-
drivers/gpu/drm/drm_gem.c | 5 +-
drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_ttm.c | 3 +-
drivers/gpu/drm/i915/gt/shmem_utils.c | 3 +-
drivers/gpu/drm/ttm/tests/ttm_tt_test.c | 2 +-
drivers/gpu/drm/ttm/ttm_backup.c | 3 +-
drivers/gpu/drm/ttm/ttm_tt.c | 2 +-
fs/xfs/scrub/xfile.c | 3 +-
fs/xfs/xfs_buf_mem.c | 2 +-
include/linux/shmem_fs.h | 8 ++-
ipc/shm.c | 6 +--
mm/memfd.c | 2 +-
mm/memfd_luo.c | 2 +-
mm/shmem.c | 59 +++++++++++++----------
security/keys/big_key.c | 2 +-
16 files changed, 57 insertions(+), 49 deletions(-)

diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c
index 9322a9287dc7..0bc36957979d 100644
--- a/arch/x86/kernel/cpu/sgx/ioctl.c
+++ b/arch/x86/kernel/cpu/sgx/ioctl.c
@@ -83,7 +83,7 @@ static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs)
encl_size = secs->size + PAGE_SIZE;

backing = shmem_file_setup("SGX backing", encl_size + (encl_size >> 5),
- VM_NORESERVE);
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(backing)) {
ret = PTR_ERR(backing);
goto err_out_shrink;
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index e4df43427394..be4dca2bc34e 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -130,14 +130,15 @@ int drm_gem_object_init_with_mnt(struct drm_device *dev,
struct vfsmount *gemfs)
{
struct file *filp;
+ const vma_flags_t flags = mk_vma_flags(VMA_NORESERVE_BIT);

drm_gem_private_object_init(dev, obj, size);

if (gemfs)
filp = shmem_file_setup_with_mnt(gemfs, "drm mm object", size,
- VM_NORESERVE);
+ flags);
else
- filp = shmem_file_setup("drm mm object", size, VM_NORESERVE);
+ filp = shmem_file_setup("drm mm object", size, flags);

if (IS_ERR(filp))
return PTR_ERR(filp);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 26dda55a07ff..fe1843497b27 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -496,7 +496,7 @@ static int __create_shmem(struct drm_i915_private *i915,
struct drm_gem_object *obj,
resource_size_t size)
{
- unsigned long flags = VM_NORESERVE;
+ const vma_flags_t flags = mk_vma_flags(VMA_NORESERVE_BIT);
struct file *filp;

drm_gem_private_object_init(&i915->drm, obj, size);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
index f65fe86c02b5..7b1a7d01db2b 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
@@ -200,7 +200,8 @@ static int i915_ttm_tt_shmem_populate(struct ttm_device *bdev,
struct address_space *mapping;
gfp_t mask;

- filp = shmem_file_setup("i915-shmem-tt", size, VM_NORESERVE);
+ filp = shmem_file_setup("i915-shmem-tt", size,
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(filp))
return PTR_ERR(filp);

diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c
index 365c4b8b04f4..5f37c699a320 100644
--- a/drivers/gpu/drm/i915/gt/shmem_utils.c
+++ b/drivers/gpu/drm/i915/gt/shmem_utils.c
@@ -19,7 +19,8 @@ struct file *shmem_create_from_data(const char *name, void *data, size_t len)
struct file *file;
int err;

- file = shmem_file_setup(name, PAGE_ALIGN(len), VM_NORESERVE);
+ file = shmem_file_setup(name, PAGE_ALIGN(len),
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(file))
return file;

diff --git a/drivers/gpu/drm/ttm/tests/ttm_tt_test.c b/drivers/gpu/drm/ttm/tests/ttm_tt_test.c
index 61ec6f580b62..bd5f7d0b9b62 100644
--- a/drivers/gpu/drm/ttm/tests/ttm_tt_test.c
+++ b/drivers/gpu/drm/ttm/tests/ttm_tt_test.c
@@ -143,7 +143,7 @@ static void ttm_tt_fini_shmem(struct kunit *test)
err = ttm_tt_init(tt, bo, 0, caching, 0);
KUNIT_ASSERT_EQ(test, err, 0);

- shmem = shmem_file_setup("ttm swap", BO_SIZE, 0);
+ shmem = shmem_file_setup("ttm swap", BO_SIZE, EMPTY_VMA_FLAGS);
tt->swap_storage = shmem;

ttm_tt_fini(tt);
diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c
index 32530c75f038..6bd4c123d94c 100644
--- a/drivers/gpu/drm/ttm/ttm_backup.c
+++ b/drivers/gpu/drm/ttm/ttm_backup.c
@@ -178,5 +178,6 @@ EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail);
*/
struct file *ttm_backup_shmem_create(loff_t size)
{
- return shmem_file_setup("ttm shmem backup", size, 0);
+ return shmem_file_setup("ttm shmem backup", size,
+ EMPTY_VMA_FLAGS);
}
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 611d20ab966d..f73a5ce87645 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -330,7 +330,7 @@ int ttm_tt_swapout(struct ttm_device *bdev, struct ttm_tt *ttm,
struct page *to_page;
int i, ret;

- swap_storage = shmem_file_setup("ttm swap", size, 0);
+ swap_storage = shmem_file_setup("ttm swap", size, EMPTY_VMA_FLAGS);
if (IS_ERR(swap_storage)) {
pr_err("Failed allocating swap storage\n");
return PTR_ERR(swap_storage);
diff --git a/fs/xfs/scrub/xfile.c b/fs/xfs/scrub/xfile.c
index c753c79df203..fe0584a39f16 100644
--- a/fs/xfs/scrub/xfile.c
+++ b/fs/xfs/scrub/xfile.c
@@ -61,7 +61,8 @@ xfile_create(
if (!xf)
return -ENOMEM;

- xf->file = shmem_kernel_file_setup(description, isize, VM_NORESERVE);
+ xf->file = shmem_kernel_file_setup(description, isize,
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(xf->file)) {
error = PTR_ERR(xf->file);
goto out_xfile;
diff --git a/fs/xfs/xfs_buf_mem.c b/fs/xfs/xfs_buf_mem.c
index dcbfa274e06d..fd6f0a5bc0ea 100644
--- a/fs/xfs/xfs_buf_mem.c
+++ b/fs/xfs/xfs_buf_mem.c
@@ -62,7 +62,7 @@ xmbuf_alloc(
if (!btp)
return -ENOMEM;

- file = shmem_kernel_file_setup(descr, 0, 0);
+ file = shmem_kernel_file_setup(descr, 0, EMPTY_VMA_FLAGS);
if (IS_ERR(file)) {
error = PTR_ERR(file);
goto out_free_btp;
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index e2069b3179c4..a8273b32e041 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -102,12 +102,10 @@ static inline struct shmem_inode_info *SHMEM_I(struct inode *inode)
extern const struct fs_parameter_spec shmem_fs_parameters[];
extern void shmem_init(void);
extern int shmem_init_fs_context(struct fs_context *fc);
-extern struct file *shmem_file_setup(const char *name,
- loff_t size, unsigned long flags);
-extern struct file *shmem_kernel_file_setup(const char *name, loff_t size,
- unsigned long flags);
+struct file *shmem_file_setup(const char *name, loff_t size, vma_flags_t flags);
+struct file *shmem_kernel_file_setup(const char *name, loff_t size, vma_flags_t vma_flags);
extern struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt,
- const char *name, loff_t size, unsigned long flags);
+ const char *name, loff_t size, vma_flags_t flags);
int shmem_zero_setup(struct vm_area_struct *vma);
int shmem_zero_setup_desc(struct vm_area_desc *desc);
extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr,
diff --git a/ipc/shm.c b/ipc/shm.c
index 2c7379c4c647..e8c7d1924c50 100644
--- a/ipc/shm.c
+++ b/ipc/shm.c
@@ -708,6 +708,7 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
struct shmid_kernel *shp;
size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
const bool has_no_reserve = shmflg & SHM_NORESERVE;
+ vma_flags_t acctflag = EMPTY_VMA_FLAGS;
struct file *file;
char name[13];

@@ -738,7 +739,6 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params)

sprintf(name, "SYSV%08x", key);
if (shmflg & SHM_HUGETLB) {
- vma_flags_t acctflag = EMPTY_VMA_FLAGS;
struct hstate *hs;
size_t hugesize;

@@ -755,14 +755,12 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
file = hugetlb_file_setup(name, hugesize, acctflag,
HUGETLB_SHMFS_INODE, (shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
} else {
- vm_flags_t acctflag = 0;
-
/*
* Do not allow no accounting for OVERCOMMIT_NEVER, even
* if it's asked for.
*/
if (has_no_reserve && sysctl_overcommit_memory != OVERCOMMIT_NEVER)
- acctflag = VM_NORESERVE;
+ vma_flags_set(&acctflag, VMA_NORESERVE_BIT);
file = shmem_kernel_file_setup(name, size, acctflag);
}
error = PTR_ERR(file);
diff --git a/mm/memfd.c b/mm/memfd.c
index 5f95f639550c..f3a8950850a2 100644
--- a/mm/memfd.c
+++ b/mm/memfd.c
@@ -469,7 +469,7 @@ static struct file *alloc_file(const char *name, unsigned int flags)
(flags >> MFD_HUGE_SHIFT) &
MFD_HUGE_MASK);
} else {
- file = shmem_file_setup(name, 0, VM_NORESERVE);
+ file = shmem_file_setup(name, 0, mk_vma_flags(VMA_NORESERVE_BIT));
}
if (IS_ERR(file))
return file;
diff --git a/mm/memfd_luo.c b/mm/memfd_luo.c
index 4f6ba63b4310..a2629dcfd0f1 100644
--- a/mm/memfd_luo.c
+++ b/mm/memfd_luo.c
@@ -443,7 +443,7 @@ static int memfd_luo_retrieve(struct liveupdate_file_op_args *args)
if (!ser)
return -EINVAL;

- file = shmem_file_setup("", 0, VM_NORESERVE);
+ file = shmem_file_setup("", 0, mk_vma_flags(VMA_NORESERVE_BIT));

if (IS_ERR(file)) {
pr_err("failed to setup file: %pe\n", file);
diff --git a/mm/shmem.c b/mm/shmem.c
index 0adde3f4df27..97a8f55c7296 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3057,9 +3057,9 @@ static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode)
}

static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
- struct super_block *sb,
- struct inode *dir, umode_t mode,
- dev_t dev, unsigned long flags)
+ struct super_block *sb,
+ struct inode *dir, umode_t mode,
+ dev_t dev, vma_flags_t flags)
{
struct inode *inode;
struct shmem_inode_info *info;
@@ -3087,7 +3087,8 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
spin_lock_init(&info->lock);
atomic_set(&info->stop_eviction, 0);
info->seals = F_SEAL_SEAL;
- info->flags = (flags & VM_NORESERVE) ? SHMEM_F_NORESERVE : 0;
+ info->flags = vma_flags_test(&flags, VMA_NORESERVE_BIT)
+ ? SHMEM_F_NORESERVE : 0;
info->i_crtime = inode_get_mtime(inode);
info->fsflags = (dir == NULL) ? 0 :
SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
@@ -3140,7 +3141,7 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
#ifdef CONFIG_TMPFS_QUOTA
static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
struct super_block *sb, struct inode *dir,
- umode_t mode, dev_t dev, unsigned long flags)
+ umode_t mode, dev_t dev, vma_flags_t flags)
{
int err;
struct inode *inode;
@@ -3166,9 +3167,9 @@ static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
return ERR_PTR(err);
}
#else
-static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
+static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
struct super_block *sb, struct inode *dir,
- umode_t mode, dev_t dev, unsigned long flags)
+ umode_t mode, dev_t dev, vma_flags_t flags)
{
return __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
}
@@ -3875,7 +3876,8 @@ shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
if (!generic_ci_validate_strict_name(dir, &dentry->d_name))
return -EINVAL;

- inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, dev, VM_NORESERVE);
+ inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, dev,
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(inode))
return PTR_ERR(inode);

@@ -3910,7 +3912,8 @@ shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
struct inode *inode;
int error;

- inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, 0, VM_NORESERVE);
+ inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, 0,
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(inode)) {
error = PTR_ERR(inode);
goto err_out;
@@ -4107,7 +4110,7 @@ static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
return -ENAMETOOLONG;

inode = shmem_get_inode(idmap, dir->i_sb, dir, S_IFLNK | 0777, 0,
- VM_NORESERVE);
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(inode))
return PTR_ERR(inode);

@@ -5108,7 +5111,8 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
#endif /* CONFIG_TMPFS_QUOTA */

inode = shmem_get_inode(&nop_mnt_idmap, sb, NULL,
- S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
+ S_IFDIR | sbinfo->mode, 0,
+ mk_vma_flags(VMA_NORESERVE_BIT));
if (IS_ERR(inode)) {
error = PTR_ERR(inode);
goto failed;
@@ -5808,7 +5812,7 @@ static inline void shmem_unacct_size(unsigned long flags, loff_t size)

static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
struct super_block *sb, struct inode *dir,
- umode_t mode, dev_t dev, unsigned long flags)
+ umode_t mode, dev_t dev, vma_flags_t flags)
{
struct inode *inode = ramfs_get_inode(sb, dir, mode, dev);
return inode ? inode : ERR_PTR(-ENOSPC);
@@ -5819,10 +5823,11 @@ static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
/* common code */

static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name,
- loff_t size, unsigned long vm_flags,
+ loff_t size, vma_flags_t flags,
unsigned int i_flags)
{
- unsigned long flags = (vm_flags & VM_NORESERVE) ? SHMEM_F_NORESERVE : 0;
+ const unsigned long shmem_flags =
+ vma_flags_test(&flags, VMA_NORESERVE_BIT) ? SHMEM_F_NORESERVE : 0;
struct inode *inode;
struct file *res;

@@ -5835,13 +5840,13 @@ static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name,
if (is_idmapped_mnt(mnt))
return ERR_PTR(-EINVAL);

- if (shmem_acct_size(flags, size))
+ if (shmem_acct_size(shmem_flags, size))
return ERR_PTR(-ENOMEM);

inode = shmem_get_inode(&nop_mnt_idmap, mnt->mnt_sb, NULL,
- S_IFREG | S_IRWXUGO, 0, vm_flags);
+ S_IFREG | S_IRWXUGO, 0, flags);
if (IS_ERR(inode)) {
- shmem_unacct_size(flags, size);
+ shmem_unacct_size(shmem_flags, size);
return ERR_CAST(inode);
}
inode->i_flags |= i_flags;
@@ -5864,9 +5869,10 @@ static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name,
* checks are provided at the key or shm level rather than the inode.
* @name: name for dentry (to be seen in /proc/<pid>/maps)
* @size: size to be set for the file
- * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
+ * @vma_flags: VMA_NORESERVE_BIT suppresses pre-accounting of the entire object size
*/
-struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
+struct file *shmem_kernel_file_setup(const char *name, loff_t size,
+ vma_flags_t flags)
{
return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
}
@@ -5878,7 +5884,7 @@ EXPORT_SYMBOL_GPL(shmem_kernel_file_setup);
* @size: size to be set for the file
* @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
*/
-struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
+struct file *shmem_file_setup(const char *name, loff_t size, vma_flags_t flags)
{
return __shmem_file_setup(shm_mnt, name, size, flags, 0);
}
@@ -5889,16 +5895,17 @@ EXPORT_SYMBOL_GPL(shmem_file_setup);
* @mnt: the tmpfs mount where the file will be created
* @name: name for dentry (to be seen in /proc/<pid>/maps)
* @size: size to be set for the file
- * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
+ * @flags: VMA_NORESERVE_BIT suppresses pre-accounting of the entire object size
*/
struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
- loff_t size, unsigned long flags)
+ loff_t size, vma_flags_t flags)
{
return __shmem_file_setup(mnt, name, size, flags, 0);
}
EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);

-static struct file *__shmem_zero_setup(unsigned long start, unsigned long end, vm_flags_t vm_flags)
+static struct file *__shmem_zero_setup(unsigned long start, unsigned long end,
+ vma_flags_t flags)
{
loff_t size = end - start;

@@ -5908,7 +5915,7 @@ static struct file *__shmem_zero_setup(unsigned long start, unsigned long end, v
* accessible to the user through its mapping, use S_PRIVATE flag to
* bypass file security, in the same way as shmem_kernel_file_setup().
*/
- return shmem_kernel_file_setup("dev/zero", size, vm_flags);
+ return shmem_kernel_file_setup("dev/zero", size, flags);
}

/**
@@ -5918,7 +5925,7 @@ static struct file *__shmem_zero_setup(unsigned long start, unsigned long end, v
*/
int shmem_zero_setup(struct vm_area_struct *vma)
{
- struct file *file = __shmem_zero_setup(vma->vm_start, vma->vm_end, vma->vm_flags);
+ struct file *file = __shmem_zero_setup(vma->vm_start, vma->vm_end, vma->flags);

if (IS_ERR(file))
return PTR_ERR(file);
@@ -5939,7 +5946,7 @@ int shmem_zero_setup(struct vm_area_struct *vma)
*/
int shmem_zero_setup_desc(struct vm_area_desc *desc)
{
- struct file *file = __shmem_zero_setup(desc->start, desc->end, desc->vm_flags);
+ struct file *file = __shmem_zero_setup(desc->start, desc->end, desc->vma_flags);

if (IS_ERR(file))
return PTR_ERR(file);
diff --git a/security/keys/big_key.c b/security/keys/big_key.c
index d46862ab90d6..268f702df380 100644
--- a/security/keys/big_key.c
+++ b/security/keys/big_key.c
@@ -103,7 +103,7 @@ int big_key_preparse(struct key_preparsed_payload *prep)
0, enckey);

/* save aligned data to file */
- file = shmem_kernel_file_setup("", enclen, 0);
+ file = shmem_kernel_file_setup("", enclen, EMPTY_VMA_FLAGS);
if (IS_ERR(file)) {
ret = PTR_ERR(file);
goto err_enckey;
--
2.52.0