fs/erofs/internal.h - third_party/linux - Git at Google


            
            /* SPDX-License-Identifier: GPL-2.0-only */

            
            /*

            
             * Copyright (C) 2017-2018 HUAWEI, Inc.

            
             *             https://www.huawei.com/

            
             * Copyright (C) 2021, Alibaba Cloud

            
             */

            
            #ifndef __EROFS_INTERNAL_H

            
            #define __EROFS_INTERNAL_H


            #include <linux/fs.h>

            
            #include <linux/dax.h>

            
            #include <linux/dcache.h>

            
            #include <linux/mm.h>

            
            #include <linux/module.h>

            
            #include <linux/pagemap.h>

            
            #include <linux/bio.h>

            
            #include <linux/magic.h>

            
            #include <linux/slab.h>

            
            #include <linux/vmalloc.h>

            
            #include <linux/iomap.h>

            
            #include "erofs_fs.h"


            __printf(2, 3) void _erofs_printk(struct super_block *sb, const char *fmt, ...);

            
            #define erofs_err(sb, fmt, ...)	\

            
            	_erofs_printk(sb, KERN_ERR fmt "\n", ##__VA_ARGS__)

            
            #define erofs_warn(sb, fmt, ...) \

            
            	_erofs_printk(sb, KERN_WARNING fmt "\n", ##__VA_ARGS__)

            
            #define erofs_info(sb, fmt, ...) \

            
            	_erofs_printk(sb, KERN_INFO fmt "\n", ##__VA_ARGS__)


            #ifdef CONFIG_EROFS_FS_DEBUG

            
            #define DBG_BUGON               BUG_ON

            
            #else

            
            #define DBG_BUGON(x)            ((void)(x))

            
            #endif	/* !CONFIG_EROFS_FS_DEBUG */


            /* EROFS_SUPER_MAGIC_V1 to represent the whole file system */

            
            #define EROFS_SUPER_MAGIC   EROFS_SUPER_MAGIC_V1


            typedef u64 erofs_nid_t;

            
            typedef u64 erofs_off_t;

            
            typedef u64 erofs_blk_t;


            struct erofs_device_info {

            
            	char *path;

            
            	struct file *file;

            
            	struct dax_device *dax_dev;

            
            	u64 fsoff, dax_part_off;


            	erofs_blk_t blocks;

            
            	erofs_blk_t uniaddr;

            
            };


            enum {

            
            	EROFS_SYNC_DECOMPRESS_AUTO,

            
            	EROFS_SYNC_DECOMPRESS_FORCE_ON,

            
            	EROFS_SYNC_DECOMPRESS_FORCE_OFF

            
            };


            struct erofs_mount_opts {

            
            	/* current strategy of how to use managed cache */

            
            	unsigned char cache_strategy;

            
            	unsigned int mount_opt;

            
            };


            struct erofs_dev_context {

            
            	struct idr tree;

            
            	struct rw_semaphore rwsem;


            	unsigned int extra_devices;

            
            	bool flatdev;

            
            };


            struct erofs_sb_lz4_info {

            
            	u16 max_pclusterblks;	/* maximum physical blocks for LZ4 pclusters */

            
            };


            struct erofs_xattr_prefix_item {

            
            	struct erofs_xattr_long_prefix *prefix;

            
            	u8 infix_len;

            
            };


            struct erofs_sb_info {

            
            	struct erofs_device_info dif0;

            
            	struct erofs_mount_opts opt;	/* options */

            
            #ifdef CONFIG_EROFS_FS_ZIP

            
            	/* list for all registered superblocks, mainly for shrinker */

            
            	struct list_head list;

            
            	struct mutex umount_mutex;


            	/* managed XArray arranged in physical block number */

            
            	struct xarray managed_pslots;


            	unsigned int sync_decompress;	/* strategy for sync decompression */

            
            	unsigned int shrinker_run_no;


            	struct erofs_sb_lz4_info lz4;

            
            #endif	/* CONFIG_EROFS_FS_ZIP */

            
            	struct inode *managed_cache; /* pseudo inode to cache physical data */

            
            	struct inode *packed_inode;

            
            	struct inode *metabox_inode;

            
            	struct erofs_dev_context *devs;

            
            	u64 total_blocks;


            	u32 meta_blkaddr;

            
            #ifdef CONFIG_EROFS_FS_XATTR

            
            	u32 xattr_blkaddr;

            
            	u32 xattr_prefix_start;

            
            	u8 xattr_prefix_count;

            
            	u8 ishare_xattr_prefix_id;

            
            	struct erofs_xattr_prefix_item *xattr_prefixes;

            
            	unsigned int xattr_filter_reserved;

            
            #endif

            
            	u16 device_id_mask;	/* valid bits of device id to be used */


            	unsigned char islotbits;	/* inode slot unit size in bit shift */

            
            	unsigned char blkszbits;	/* filesystem block size in bit shift */


            	u32 sb_size;			/* total superblock size */

            
            	u32 fixed_nsec;

            
            	s64 epoch;


            	/* what we really care is nid, rather than ino.. */

            
            	erofs_nid_t root_nid;

            
            	erofs_nid_t packed_nid;

            
            	erofs_nid_t metabox_nid;

            
            	/* used for statfs, f_files - f_favail */

            
            	u64 inos;


            	char *volume_name;

            
            	u32 feature_compat;

            
            	u32 feature_incompat;

            
            	u16 available_compr_algs;


            	/* sysfs support */

            
            	struct kobject s_kobj;		/* /sys/fs/erofs/<devname> */

            
            	struct completion s_kobj_unregister;

            
            	erofs_off_t dir_ra_bytes;


            	char *domain_id;

            
            };


            #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)

            
            #define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)


            /* Mount flags set via mount options or defaults */

            
            #define EROFS_MOUNT_XATTR_USER		0x00000010

            
            #define EROFS_MOUNT_POSIX_ACL		0x00000020

            
            #define EROFS_MOUNT_DAX_ALWAYS		0x00000040

            
            #define EROFS_MOUNT_DAX_NEVER		0x00000080

            
            #define EROFS_MOUNT_DIRECT_IO		0x00000100

            
            #define EROFS_MOUNT_INODE_SHARE		0x00000200


            #define clear_opt(opt, option)	((opt)->mount_opt &= ~EROFS_MOUNT_##option)

            
            #define set_opt(opt, option)	((opt)->mount_opt |= EROFS_MOUNT_##option)

            
            #define test_opt(opt, option)	((opt)->mount_opt & EROFS_MOUNT_##option)


            static inline bool erofs_is_fileio_mode(struct erofs_sb_info *sbi)

            
            {

            
            	return IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && sbi->dif0.file;

            
            }


            extern struct file_system_type erofs_anon_fs_type;


            enum {

            
            	EROFS_ZIP_CACHE_DISABLED,

            
            	EROFS_ZIP_CACHE_READAHEAD,

            
            	EROFS_ZIP_CACHE_READAROUND

            
            };


            struct erofs_buf {

            
            	struct address_space *mapping;

            
            	u64 off;

            
            	struct page *page;

            
            	void *base;

            
            	bool mc;

            
            };

            
            #define __EROFS_BUF_INITIALIZER	((struct erofs_buf){ .page = NULL })


            #define erofs_blknr(sb, pos)	((erofs_blk_t)((pos) >> (sb)->s_blocksize_bits))

            
            #define erofs_blkoff(sb, pos)	((pos) & ((sb)->s_blocksize - 1))

            
            #define erofs_pos(sb, blk)	((erofs_off_t)(blk) << (sb)->s_blocksize_bits)

            
            #define erofs_iblks(i)	(round_up((i)->i_size, i_blocksize(i)) >> (i)->i_blkbits)


            #define EROFS_FEATURE_FUNCS(name, compat, feature) \

            
            static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \

            
            { \

            
            	return sbi->feature_##compat & EROFS_FEATURE_##feature; \

            
            }


            EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)

            
            EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)

            
            EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)

            
            EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)

            
            EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)

            
            EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)

            
            EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)

            
            EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)

            
            EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)

            
            EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)

            
            EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT)

            
            EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX)

            
            EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)

            
            EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)

            
            EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX)

            
            EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX)

            
            EROFS_FEATURE_FUNCS(ishare_xattrs, compat, COMPAT_ISHARE_XATTRS)


            static inline u64 erofs_nid_to_ino64(struct erofs_sb_info *sbi, erofs_nid_t nid)

            
            {

            
            	if (!erofs_sb_has_metabox(sbi))

            
            		return nid;


            	/*

            
            	 * When metadata compression is enabled, avoid generating excessively

            
            	 * large inode numbers for metadata-compressed inodes.  Shift NIDs in

            
            	 * the 31-62 bit range left by one and move the metabox flag to bit 31.

            
            	 *

            
            	 * Note: on-disk NIDs remain unchanged as they are primarily used for

            
            	 * compatibility with non-LFS 32-bit applications.

            
            	 */

            
            	return ((nid << 1) & GENMASK_ULL(63, 32)) | (nid & GENMASK(30, 0)) |

            
            		((nid >> EROFS_DIRENT_NID_METABOX_BIT) << 31);

            
            }


            /* atomic flag definitions */

            
            #define EROFS_I_EA_INITED_BIT	0

            
            #define EROFS_I_Z_INITED_BIT	1


            /* bitlock definitions (arranged in reverse order) */

            
            #define EROFS_I_BL_XATTR_BIT	(BITS_PER_LONG - 1)

            
            #define EROFS_I_BL_Z_BIT	(BITS_PER_LONG - 2)


            /* default readahead size of directories */

            
            #define EROFS_DIR_RA_BYTES	16384


            struct erofs_inode_fingerprint {

            
            	u8 *opaque;

            
            	int size;

            
            };


            struct erofs_inode {

            
            	erofs_nid_t nid;


            	/* atomic flags (including bitlocks) */

            
            	unsigned long flags;


            	unsigned char datalayout;

            
            	unsigned char inode_isize;

            
            	bool dot_omitted;

            
            	unsigned int xattr_isize;


            	unsigned int xattr_name_filter;

            
            	unsigned int xattr_shared_count;

            
            	unsigned int *xattr_shared_xattrs;


            	union {

            
            		erofs_blk_t startblk;

            
            		struct {

            
            			unsigned short	chunkformat;

            
            			unsigned char	chunkbits;

            
            		};

            
            #ifdef CONFIG_EROFS_FS_ZIP

            
            		struct {

            
            			unsigned short z_advise;

            
            			unsigned char  z_algofmt[2];

            
            			unsigned char  z_lclusterbits;

            
            			union {

            
            				u64    z_tailextent_headlcn;

            
            				u64    z_extents;

            
            			};

            
            			erofs_off_t    z_fragmentoff;

            
            			unsigned short z_idata_size;

            
            		};

            
            #endif	/* CONFIG_EROFS_FS_ZIP */

            
            	};

            
            #ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE

            
            	struct list_head ishare_list;

            
            	union {

            
            		/* for each anon shared inode */

            
            		struct {

            
            			struct erofs_inode_fingerprint fingerprint;

            
            			spinlock_t ishare_lock;

            
            		};

            
            		/* for each real filesystem inode */

            
            		struct dentry *sharedentry;

            
            	};

            
            #endif

            
            	/* the corresponding vfs inode */

            
            	struct inode vfs_inode;

            
            };


            #define EROFS_I(ptr)	container_of(ptr, struct erofs_inode, vfs_inode)


            static inline bool erofs_inode_in_metabox(struct inode *inode)

            
            {

            
            	return EROFS_I(inode)->nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT);

            
            }


            static inline erofs_off_t erofs_iloc(struct inode *inode)

            
            {

            
            	struct erofs_sb_info *sbi = EROFS_I_SB(inode);

            
            	erofs_nid_t nid_lo = EROFS_I(inode)->nid & EROFS_DIRENT_NID_MASK;


            	if (erofs_inode_in_metabox(inode))

            
            		return nid_lo << sbi->islotbits;

            
            	return erofs_pos(inode->i_sb, sbi->meta_blkaddr) +

            
            		(nid_lo << sbi->islotbits);

            
            }


            static inline unsigned int erofs_inode_version(unsigned int ifmt)

            
            {

            
            	return (ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK;

            
            }


            static inline unsigned int erofs_inode_datalayout(unsigned int ifmt)

            
            {

            
            	return (ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK;

            
            }


            /* reclaiming is never triggered when allocating new folios. */

            
            static inline struct folio *erofs_grab_folio_nowait(struct address_space *as,

            
            						    pgoff_t index)

            
            {

            
            	return __filemap_get_folio(as, index,

            
            			FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,

            
            			readahead_gfp_mask(as) & ~__GFP_RECLAIM);

            
            }


            /* Allocated on disk at @m_pa (e.g. NOT a fragment extent) */

            
            #define EROFS_MAP_MAPPED		0x0001

            
            /* Located in metadata (could be copied from bd_inode) */

            
            #define EROFS_MAP_META			0x0002

            
            /* @m_llen may be truncated by the runtime compared to the on-disk record */

            
            #define EROFS_MAP_PARTIAL_MAPPED	0x0004

            
            /* The on-disk @m_llen may cover only part of the encoded data */

            
            #define EROFS_MAP_PARTIAL_REF		0x0008

            
            /* Located in the special packed inode */

            
            #define EROFS_MAP_FRAGMENT		0x0010

            
            /* The encoded on-disk data will be fully handled (decompressed) */

            
            #define EROFS_MAP_FULL(f)	(!((f) & (EROFS_MAP_PARTIAL_MAPPED | \

            
            					  EROFS_MAP_PARTIAL_REF)))


            struct erofs_map_blocks {

            
            	struct erofs_buf buf;


            	erofs_off_t m_pa, m_la;

            
            	u64 m_plen, m_llen;


            	unsigned short m_deviceid;

            
            	char m_algorithmformat;

            
            	unsigned int m_flags;

            
            };


            /*

            
             * Used to get the exact decompressed length, e.g. fiemap (consider lookback

            
             * approach instead if possible since it's more metadata lightweight.)

            
             */

            
            #define EROFS_GET_BLOCKS_FIEMAP		0x0001

            
            /* Used to map the whole extent if non-negligible data is requested for LZMA */

            
            #define EROFS_GET_BLOCKS_READMORE	0x0002

            
            /* Used to map tail extent for tailpacking inline or fragment pcluster */

            
            #define EROFS_GET_BLOCKS_FINDTAIL	0x0004


            enum {

            
            	Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,

            
            	Z_EROFS_COMPRESSION_INTERLACED,

            
            	Z_EROFS_COMPRESSION_RUNTIME_MAX

            
            };


            struct erofs_map_dev {

            
            	struct super_block *m_sb;

            
            	struct erofs_device_info *m_dif;

            
            	struct block_device *m_bdev;


            	erofs_off_t m_pa;

            
            	unsigned int m_deviceid;

            
            };


            extern const struct super_operations erofs_sops;

            
            extern const struct address_space_operations erofs_aops;

            
            extern const struct address_space_operations erofs_fileio_aops;

            
            extern const struct address_space_operations z_erofs_aops;


            extern const struct inode_operations erofs_generic_iops;

            
            extern const struct inode_operations erofs_symlink_iops;

            
            extern const struct inode_operations erofs_fast_symlink_iops;

            
            extern const struct inode_operations erofs_dir_iops;


            extern const struct file_operations erofs_file_fops;

            
            extern const struct file_operations erofs_dir_fops;

            
            extern const struct file_operations erofs_ishare_fops;


            extern const struct iomap_ops z_erofs_iomap_report_ops;


            int erofs_setup_managed_cache(struct super_block *sb);

            
            #ifdef CONFIG_EROFS_FS_BACKED_BY_FILE

            
            int erofs_read_meta_folio(struct file *file, struct folio *folio);

            
            #else

            
            #define erofs_read_meta_folio NULL

            
            #endif

            
            void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf,

            
            			  erofs_off_t *offset, int *lengthp);

            
            void erofs_unmap_metabuf(struct erofs_buf *buf);

            
            void erofs_put_metabuf(struct erofs_buf *buf);

            
            void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap);

            
            int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb,

            
            		       bool in_metabox);

            
            void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb,

            
            			 erofs_off_t offset, bool in_metabox);

            
            int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *dev);

            
            int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,

            
            		 u64 start, u64 len);

            
            loff_t erofs_file_llseek(struct file *file, loff_t offset, int whence);

            
            int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map);

            
            void erofs_onlinefolio_init(struct folio *folio);

            
            void erofs_onlinefolio_split(struct folio *folio);

            
            void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty);

            
            struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid);

            
            int erofs_getattr(struct mnt_idmap *idmap, const struct path *path,

            
            		  struct kstat *stat, u32 request_mask,

            
            		  unsigned int query_flags);

            
            int erofs_namei(struct inode *dir, const struct qstr *name,

            
            		erofs_nid_t *nid, unsigned int *d_type);


            static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)

            
            {

            
            	int retried = 0;


            	while (1) {

            
            		void *p = vm_map_ram(pages, count, -1);


            		/* retry two more times (totally 3 times) */

            
            		if (p || ++retried >= 3)

            
            			return p;

            
            		vm_unmap_aliases();

            
            	}

            
            	return NULL;

            
            }


            static inline const struct address_space_operations *

            
            erofs_get_aops(struct inode *realinode)

            
            {

            
            	if (erofs_inode_is_data_compressed(EROFS_I(realinode)->datalayout)) {

            
            		if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP))

            
            			return ERR_PTR(-EOPNOTSUPP);

            
            		DO_ONCE_LITE_IF(realinode->i_blkbits != PAGE_SHIFT,

            
            			  erofs_info, realinode->i_sb,

            
            			  "EXPERIMENTAL EROFS subpage compressed block support in use. Use at your own risk!");

            
            		return &z_erofs_aops;

            
            	}

            
            	if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) &&

            
            	    erofs_is_fileio_mode(EROFS_SB(realinode->i_sb)))

            
            		return &erofs_fileio_aops;

            
            	return &erofs_aops;

            
            }


            int erofs_register_sysfs(struct super_block *sb);

            
            void erofs_unregister_sysfs(struct super_block *sb);

            
            int __init erofs_init_sysfs(void);

            
            void erofs_exit_sysfs(void);


            struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv);

            
            static inline struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)

            
            {

            
            	return __erofs_allocpage(pagepool, gfp, false);

            
            }

            
            static inline void erofs_pagepool_add(struct page **pagepool, struct page *page)

            
            {

            
            	set_page_private(page, (unsigned long)*pagepool);

            
            	*pagepool = page;

            
            }

            
            void erofs_release_pages(struct page **pagepool);


            #ifdef CONFIG_EROFS_FS_ZIP

            
            #define MNGD_MAPPING(sbi)	((sbi)->managed_cache->i_mapping)


            extern atomic_long_t erofs_global_shrink_cnt;

            
            void erofs_shrinker_register(struct super_block *sb);

            
            void erofs_shrinker_unregister(struct super_block *sb);

            
            int __init erofs_init_shrinker(void);

            
            void erofs_exit_shrinker(void);

            
            int __init z_erofs_init_subsystem(void);

            
            void z_erofs_exit_subsystem(void);

            
            int z_erofs_init_super(struct super_block *sb);

            
            unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi,

            
            				  unsigned long nr_shrink);

            
            int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,

            
            			    int flags);

            
            void *z_erofs_get_gbuf(unsigned int requiredpages);

            
            void z_erofs_put_gbuf(void *ptr);

            
            int z_erofs_gbuf_growsize(unsigned int nrpages);

            
            int __init z_erofs_gbuf_init(void);

            
            void z_erofs_gbuf_exit(void);

            
            #else

            
            static inline void erofs_shrinker_register(struct super_block *sb) {}

            
            static inline void erofs_shrinker_unregister(struct super_block *sb) {}

            
            static inline int erofs_init_shrinker(void) { return 0; }

            
            static inline void erofs_exit_shrinker(void) {}

            
            static inline int z_erofs_init_subsystem(void) { return 0; }

            
            static inline void z_erofs_exit_subsystem(void) {}

            
            static inline int z_erofs_init_super(struct super_block *sb) { return 0; }

            
            #endif	/* !CONFIG_EROFS_FS_ZIP */

            
            int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb);


            #ifdef CONFIG_EROFS_FS_BACKED_BY_FILE

            
            struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev);

            
            void erofs_fileio_submit_bio(struct bio *bio);

            
            #else

            
            static inline struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) { return NULL; }

            
            static inline void erofs_fileio_submit_bio(struct bio *bio) {}

            
            #endif


            #ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE

            
            int __init erofs_init_ishare(void);

            
            void erofs_exit_ishare(void);

            
            bool erofs_ishare_fill_inode(struct inode *inode);

            
            void erofs_ishare_free_inode(struct inode *inode);

            
            struct inode *erofs_real_inode(struct inode *inode, bool *need_iput);

            
            #else

            
            static inline int erofs_init_ishare(void) { return 0; }

            
            static inline void erofs_exit_ishare(void) {}

            
            static inline bool erofs_ishare_fill_inode(struct inode *inode) { return false; }

            
            static inline void erofs_ishare_free_inode(struct inode *inode) {}

            
            static inline struct inode *erofs_real_inode(struct inode *inode, bool *need_iput)

            
            {

            
            	*need_iput = false;

            
            	return inode;

            
            }

            
            #endif


            long erofs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);

            
            long erofs_compat_ioctl(struct file *filp, unsigned int cmd,

            
            			unsigned long arg);


            #endif	/* __EROFS_INTERNAL_H */
	/* SPDX-License-Identifier: GPL-2.0-only */
	/*
	* Copyright (C) 2017-2018 HUAWEI, Inc.
	* https://www.huawei.com/
	* Copyright (C) 2021, Alibaba Cloud
	*/
	#ifndef __EROFS_INTERNAL_H
	#define __EROFS_INTERNAL_H

	#include <linux/fs.h>
	#include <linux/dax.h>
	#include <linux/dcache.h>
	#include <linux/mm.h>
	#include <linux/module.h>
	#include <linux/pagemap.h>
	#include <linux/bio.h>
	#include <linux/magic.h>
	#include <linux/slab.h>
	#include <linux/vmalloc.h>
	#include <linux/iomap.h>
	#include "erofs_fs.h"

	__printf(2, 3) void _erofs_printk(struct super_block sb, const char fmt, ...);
	#define erofs_err(sb, fmt, ...) \
	_erofs_printk(sb, KERN_ERR fmt "\n", ##__VA_ARGS__)
	#define erofs_warn(sb, fmt, ...) \
	_erofs_printk(sb, KERN_WARNING fmt "\n", ##__VA_ARGS__)
	#define erofs_info(sb, fmt, ...) \
	_erofs_printk(sb, KERN_INFO fmt "\n", ##__VA_ARGS__)

	#ifdef CONFIG_EROFS_FS_DEBUG
	#define DBG_BUGON BUG_ON
	#else
	#define DBG_BUGON(x) ((void)(x))
	#endif /* !CONFIG_EROFS_FS_DEBUG */

	/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */
	#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1

	typedef u64 erofs_nid_t;
	typedef u64 erofs_off_t;
	typedef u64 erofs_blk_t;

	struct erofs_device_info {
	char *path;
	struct file *file;
	struct dax_device *dax_dev;
	u64 fsoff, dax_part_off;

	erofs_blk_t blocks;
	erofs_blk_t uniaddr;
	};

	enum {
	EROFS_SYNC_DECOMPRESS_AUTO,
	EROFS_SYNC_DECOMPRESS_FORCE_ON,
	EROFS_SYNC_DECOMPRESS_FORCE_OFF
	};

	struct erofs_mount_opts {
	/* current strategy of how to use managed cache */
	unsigned char cache_strategy;
	unsigned int mount_opt;
	};

	struct erofs_dev_context {
	struct idr tree;
	struct rw_semaphore rwsem;

	unsigned int extra_devices;
	bool flatdev;
	};

	struct erofs_sb_lz4_info {
	u16 max_pclusterblks; /* maximum physical blocks for LZ4 pclusters */
	};

	struct erofs_xattr_prefix_item {
	struct erofs_xattr_long_prefix *prefix;
	u8 infix_len;
	};

	struct erofs_sb_info {
	struct erofs_device_info dif0;
	struct erofs_mount_opts opt; /* options */
	#ifdef CONFIG_EROFS_FS_ZIP
	/* list for all registered superblocks, mainly for shrinker */
	struct list_head list;
	struct mutex umount_mutex;

	/* managed XArray arranged in physical block number */
	struct xarray managed_pslots;

	unsigned int sync_decompress; /* strategy for sync decompression */
	unsigned int shrinker_run_no;

	struct erofs_sb_lz4_info lz4;
	#endif /* CONFIG_EROFS_FS_ZIP */
	struct inode managed_cache; / pseudo inode to cache physical data */
	struct inode *packed_inode;
	struct inode *metabox_inode;
	struct erofs_dev_context *devs;
	u64 total_blocks;

	u32 meta_blkaddr;
	#ifdef CONFIG_EROFS_FS_XATTR
	u32 xattr_blkaddr;
	u32 xattr_prefix_start;
	u8 xattr_prefix_count;
	u8 ishare_xattr_prefix_id;
	struct erofs_xattr_prefix_item *xattr_prefixes;
	unsigned int xattr_filter_reserved;
	#endif
	u16 device_id_mask; /* valid bits of device id to be used */

	unsigned char islotbits; /* inode slot unit size in bit shift */
	unsigned char blkszbits; /* filesystem block size in bit shift */

	u32 sb_size; /* total superblock size */
	u32 fixed_nsec;
	s64 epoch;

	/* what we really care is nid, rather than ino.. */
	erofs_nid_t root_nid;
	erofs_nid_t packed_nid;
	erofs_nid_t metabox_nid;
	/* used for statfs, f_files - f_favail */
	u64 inos;

	char *volume_name;
	u32 feature_compat;
	u32 feature_incompat;
	u16 available_compr_algs;

	/* sysfs support */
	struct kobject s_kobj; /* /sys/fs/erofs/<devname> */
	struct completion s_kobj_unregister;
	erofs_off_t dir_ra_bytes;

	char *domain_id;
	};

	#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
	#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)

	/* Mount flags set via mount options or defaults */
	#define EROFS_MOUNT_XATTR_USER 0x00000010
	#define EROFS_MOUNT_POSIX_ACL 0x00000020
	#define EROFS_MOUNT_DAX_ALWAYS 0x00000040
	#define EROFS_MOUNT_DAX_NEVER 0x00000080
	#define EROFS_MOUNT_DIRECT_IO 0x00000100
	#define EROFS_MOUNT_INODE_SHARE 0x00000200

	#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option)
	#define set_opt(opt, option) ((opt)->mount_opt \|= EROFS_MOUNT_##option)
	#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option)

	static inline bool erofs_is_fileio_mode(struct erofs_sb_info *sbi)
	{
	return IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && sbi->dif0.file;
	}

	extern struct file_system_type erofs_anon_fs_type;

	enum {
	EROFS_ZIP_CACHE_DISABLED,
	EROFS_ZIP_CACHE_READAHEAD,
	EROFS_ZIP_CACHE_READAROUND
	};

	struct erofs_buf {
	struct address_space *mapping;
	u64 off;
	struct page *page;
	void *base;
	bool mc;
	};
	#define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL })

	#define erofs_blknr(sb, pos) ((erofs_blk_t)((pos) >> (sb)->s_blocksize_bits))
	#define erofs_blkoff(sb, pos) ((pos) & ((sb)->s_blocksize - 1))
	#define erofs_pos(sb, blk) ((erofs_off_t)(blk) << (sb)->s_blocksize_bits)
	#define erofs_iblks(i) (round_up((i)->i_size, i_blocksize(i)) >> (i)->i_blkbits)

	#define EROFS_FEATURE_FUNCS(name, compat, feature) \
	static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
	{ \
	return sbi->feature_##compat & EROFS_FEATURE_##feature; \
	}

	EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
	EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
	EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
	EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
	EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
	EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)
	EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
	EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)
	EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
	EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)
	EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT)
	EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX)
	EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
	EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)
	EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX)
	EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX)
	EROFS_FEATURE_FUNCS(ishare_xattrs, compat, COMPAT_ISHARE_XATTRS)

	static inline u64 erofs_nid_to_ino64(struct erofs_sb_info *sbi, erofs_nid_t nid)
	{
	if (!erofs_sb_has_metabox(sbi))
	return nid;

	/*
	* When metadata compression is enabled, avoid generating excessively
	* large inode numbers for metadata-compressed inodes. Shift NIDs in
	* the 31-62 bit range left by one and move the metabox flag to bit 31.
	*
	* Note: on-disk NIDs remain unchanged as they are primarily used for
	* compatibility with non-LFS 32-bit applications.
	*/
	return ((nid << 1) & GENMASK_ULL(63, 32)) \| (nid & GENMASK(30, 0)) \|
	((nid >> EROFS_DIRENT_NID_METABOX_BIT) << 31);
	}

	/* atomic flag definitions */
	#define EROFS_I_EA_INITED_BIT 0
	#define EROFS_I_Z_INITED_BIT 1

	/* bitlock definitions (arranged in reverse order) */
	#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)
	#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)

	/* default readahead size of directories */
	#define EROFS_DIR_RA_BYTES 16384

	struct erofs_inode_fingerprint {
	u8 *opaque;
	int size;
	};

	struct erofs_inode {
	erofs_nid_t nid;

	/* atomic flags (including bitlocks) */
	unsigned long flags;

	unsigned char datalayout;
	unsigned char inode_isize;
	bool dot_omitted;
	unsigned int xattr_isize;

	unsigned int xattr_name_filter;
	unsigned int xattr_shared_count;
	unsigned int *xattr_shared_xattrs;

	union {
	erofs_blk_t startblk;
	struct {
	unsigned short chunkformat;
	unsigned char chunkbits;
	};
	#ifdef CONFIG_EROFS_FS_ZIP
	struct {
	unsigned short z_advise;
	unsigned char z_algofmt[2];
	unsigned char z_lclusterbits;
	union {
	u64 z_tailextent_headlcn;
	u64 z_extents;
	};
	erofs_off_t z_fragmentoff;
	unsigned short z_idata_size;
	};
	#endif /* CONFIG_EROFS_FS_ZIP */
	};
	#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE
	struct list_head ishare_list;
	union {
	/* for each anon shared inode */
	struct {
	struct erofs_inode_fingerprint fingerprint;
	spinlock_t ishare_lock;
	};
	/* for each real filesystem inode */
	struct dentry *sharedentry;
	};
	#endif
	/* the corresponding vfs inode */
	struct inode vfs_inode;
	};

	#define EROFS_I(ptr) container_of(ptr, struct erofs_inode, vfs_inode)

	static inline bool erofs_inode_in_metabox(struct inode *inode)
	{
	return EROFS_I(inode)->nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT);
	}

	static inline erofs_off_t erofs_iloc(struct inode *inode)
	{
	struct erofs_sb_info *sbi = EROFS_I_SB(inode);
	erofs_nid_t nid_lo = EROFS_I(inode)->nid & EROFS_DIRENT_NID_MASK;

	if (erofs_inode_in_metabox(inode))
	return nid_lo << sbi->islotbits;
	return erofs_pos(inode->i_sb, sbi->meta_blkaddr) +
	(nid_lo << sbi->islotbits);
	}

	static inline unsigned int erofs_inode_version(unsigned int ifmt)
	{
	return (ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK;
	}

	static inline unsigned int erofs_inode_datalayout(unsigned int ifmt)
	{
	return (ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK;
	}

	/* reclaiming is never triggered when allocating new folios. */
	static inline struct folio erofs_grab_folio_nowait(struct address_space as,
	pgoff_t index)
	{
	return __filemap_get_folio(as, index,
	FGP_LOCK\|FGP_CREAT\|FGP_NOFS\|FGP_NOWAIT,
	readahead_gfp_mask(as) & ~__GFP_RECLAIM);
	}

	/* Allocated on disk at @m_pa (e.g. NOT a fragment extent) */
	#define EROFS_MAP_MAPPED 0x0001
	/* Located in metadata (could be copied from bd_inode) */
	#define EROFS_MAP_META 0x0002
	/* @m_llen may be truncated by the runtime compared to the on-disk record */
	#define EROFS_MAP_PARTIAL_MAPPED 0x0004
	/* The on-disk @m_llen may cover only part of the encoded data */
	#define EROFS_MAP_PARTIAL_REF 0x0008
	/* Located in the special packed inode */
	#define EROFS_MAP_FRAGMENT 0x0010
	/* The encoded on-disk data will be fully handled (decompressed) */
	#define EROFS_MAP_FULL(f) (!((f) & (EROFS_MAP_PARTIAL_MAPPED \| \
	EROFS_MAP_PARTIAL_REF)))

	struct erofs_map_blocks {
	struct erofs_buf buf;

	erofs_off_t m_pa, m_la;
	u64 m_plen, m_llen;

	unsigned short m_deviceid;
	char m_algorithmformat;
	unsigned int m_flags;
	};

	/*
	* Used to get the exact decompressed length, e.g. fiemap (consider lookback
	* approach instead if possible since it's more metadata lightweight.)
	*/
	#define EROFS_GET_BLOCKS_FIEMAP 0x0001
	/* Used to map the whole extent if non-negligible data is requested for LZMA */
	#define EROFS_GET_BLOCKS_READMORE 0x0002
	/* Used to map tail extent for tailpacking inline or fragment pcluster */
	#define EROFS_GET_BLOCKS_FINDTAIL 0x0004

	enum {
	Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
	Z_EROFS_COMPRESSION_INTERLACED,
	Z_EROFS_COMPRESSION_RUNTIME_MAX
	};

	struct erofs_map_dev {
	struct super_block *m_sb;
	struct erofs_device_info *m_dif;
	struct block_device *m_bdev;

	erofs_off_t m_pa;
	unsigned int m_deviceid;
	};

	extern const struct super_operations erofs_sops;
	extern const struct address_space_operations erofs_aops;
	extern const struct address_space_operations erofs_fileio_aops;
	extern const struct address_space_operations z_erofs_aops;

	extern const struct inode_operations erofs_generic_iops;
	extern const struct inode_operations erofs_symlink_iops;
	extern const struct inode_operations erofs_fast_symlink_iops;
	extern const struct inode_operations erofs_dir_iops;

	extern const struct file_operations erofs_file_fops;
	extern const struct file_operations erofs_dir_fops;
	extern const struct file_operations erofs_ishare_fops;

	extern const struct iomap_ops z_erofs_iomap_report_ops;

	int erofs_setup_managed_cache(struct super_block *sb);
	#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE
	int erofs_read_meta_folio(struct file file, struct folio folio);
	#else
	#define erofs_read_meta_folio NULL
	#endif
	void erofs_read_metadata(struct super_block sb, struct erofs_buf *buf,
	erofs_off_t offset, int lengthp);
	void erofs_unmap_metabuf(struct erofs_buf *buf);
	void erofs_put_metabuf(struct erofs_buf *buf);
	void erofs_bread(struct erofs_buf buf, erofs_off_t offset, bool need_kmap);
	int erofs_init_metabuf(struct erofs_buf buf, struct super_block sb,
	bool in_metabox);
	void erofs_read_metabuf(struct erofs_buf buf, struct super_block *sb,
	erofs_off_t offset, bool in_metabox);
	int erofs_map_dev(struct super_block sb, struct erofs_map_dev dev);
	int erofs_fiemap(struct inode inode, struct fiemap_extent_info fieinfo,
	u64 start, u64 len);
	loff_t erofs_file_llseek(struct file *file, loff_t offset, int whence);
	int erofs_map_blocks(struct inode inode, struct erofs_map_blocks map);
	void erofs_onlinefolio_init(struct folio *folio);
	void erofs_onlinefolio_split(struct folio *folio);
	void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty);
	struct inode erofs_iget(struct super_block sb, erofs_nid_t nid);
	int erofs_getattr(struct mnt_idmap idmap, const struct path path,
	struct kstat *stat, u32 request_mask,
	unsigned int query_flags);
	int erofs_namei(struct inode dir, const struct qstr name,
	erofs_nid_t nid, unsigned int d_type);

	static inline void erofs_vm_map_ram(struct page *pages, unsigned int count)
	{
	int retried = 0;

	while (1) {
	void *p = vm_map_ram(pages, count, -1);

	/* retry two more times (totally 3 times) */
	if (p \|\| ++retried >= 3)
	return p;
	vm_unmap_aliases();
	}
	return NULL;
	}

	static inline const struct address_space_operations *
	erofs_get_aops(struct inode *realinode)
	{
	if (erofs_inode_is_data_compressed(EROFS_I(realinode)->datalayout)) {
	if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP))
	return ERR_PTR(-EOPNOTSUPP);
	DO_ONCE_LITE_IF(realinode->i_blkbits != PAGE_SHIFT,
	erofs_info, realinode->i_sb,
	"EXPERIMENTAL EROFS subpage compressed block support in use. Use at your own risk!");
	return &z_erofs_aops;
	}
	if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) &&
	erofs_is_fileio_mode(EROFS_SB(realinode->i_sb)))
	return &erofs_fileio_aops;
	return &erofs_aops;
	}

	int erofs_register_sysfs(struct super_block *sb);
	void erofs_unregister_sysfs(struct super_block *sb);
	int __init erofs_init_sysfs(void);
	void erofs_exit_sysfs(void);

	struct page __erofs_allocpage(struct page *pagepool, gfp_t gfp, bool tryrsv);
	static inline struct page erofs_allocpage(struct page *pagepool, gfp_t gfp)
	{
	return __erofs_allocpage(pagepool, gfp, false);
	}
	static inline void erofs_pagepool_add(struct page *pagepool, struct page page)
	{
	set_page_private(page, (unsigned long)*pagepool);
	*pagepool = page;
	}
	void erofs_release_pages(struct page **pagepool);

	#ifdef CONFIG_EROFS_FS_ZIP
	#define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping)

	extern atomic_long_t erofs_global_shrink_cnt;
	void erofs_shrinker_register(struct super_block *sb);
	void erofs_shrinker_unregister(struct super_block *sb);
	int __init erofs_init_shrinker(void);
	void erofs_exit_shrinker(void);
	int __init z_erofs_init_subsystem(void);
	void z_erofs_exit_subsystem(void);
	int z_erofs_init_super(struct super_block *sb);
	unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi,
	unsigned long nr_shrink);
	int z_erofs_map_blocks_iter(struct inode inode, struct erofs_map_blocks map,
	int flags);
	void *z_erofs_get_gbuf(unsigned int requiredpages);
	void z_erofs_put_gbuf(void *ptr);
	int z_erofs_gbuf_growsize(unsigned int nrpages);
	int __init z_erofs_gbuf_init(void);
	void z_erofs_gbuf_exit(void);
	#else
	static inline void erofs_shrinker_register(struct super_block *sb) {}
	static inline void erofs_shrinker_unregister(struct super_block *sb) {}
	static inline int erofs_init_shrinker(void) { return 0; }
	static inline void erofs_exit_shrinker(void) {}
	static inline int z_erofs_init_subsystem(void) { return 0; }
	static inline void z_erofs_exit_subsystem(void) {}
	static inline int z_erofs_init_super(struct super_block *sb) { return 0; }
	#endif /* !CONFIG_EROFS_FS_ZIP */
	int z_erofs_parse_cfgs(struct super_block sb, struct erofs_super_block dsb);

	#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE
	struct bio erofs_fileio_bio_alloc(struct erofs_map_dev mdev);
	void erofs_fileio_submit_bio(struct bio *bio);
	#else
	static inline struct bio erofs_fileio_bio_alloc(struct erofs_map_dev mdev) { return NULL; }
	static inline void erofs_fileio_submit_bio(struct bio *bio) {}
	#endif

	#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE
	int __init erofs_init_ishare(void);
	void erofs_exit_ishare(void);
	bool erofs_ishare_fill_inode(struct inode *inode);
	void erofs_ishare_free_inode(struct inode *inode);
	struct inode erofs_real_inode(struct inode inode, bool *need_iput);
	#else
	static inline int erofs_init_ishare(void) { return 0; }
	static inline void erofs_exit_ishare(void) {}
	static inline bool erofs_ishare_fill_inode(struct inode *inode) { return false; }
	static inline void erofs_ishare_free_inode(struct inode *inode) {}
	static inline struct inode erofs_real_inode(struct inode inode, bool *need_iput)
	{
	*need_iput = false;
	return inode;
	}
	#endif

	long erofs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
	long erofs_compat_ioctl(struct file *filp, unsigned int cmd,
	unsigned long arg);

	#endif /* __EROFS_INTERNAL_H */