Skip to content

Commit 4808641

Browse files
shodanshokamotin
shodanshok
authored and
amotin
committed
enforce arc_dnode_limit
Linux kernel shrinker in the context of null/root memcg does not scan dentry and inode caches added by a task running in non-root memcg. For ZFS this means that dnode cache routinely overflows, evicting valuable meta/data and putting additional memory pressure on the system. This patch restores zfs_prune_aliases as fallback when the kernel shrinker does nothing, enabling zfs to actually free dnodes. Moreover, it (indirectly) calls arc_evict when dnode_size > dnode_limit. Reviewed-by: Rob Norris <robn@despairlabs.com> Reviewed-by: Alexander Motin <mav@FreeBSD.org> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Gionatan Danti <g.danti@assyoma.it> Closes openzfs#17487 Closes openzfs#17542
1 parent 30fa92b commit 4808641

3 files changed

Lines changed: 78 additions & 11 deletions

File tree

‎include/sys/arc_impl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -954,7 +954,7 @@ typedef struct arc_sums {
954954
wmsum_t arcstat_data_size;
955955
wmsum_t arcstat_metadata_size;
956956
wmsum_t arcstat_dbuf_size;
957-
wmsum_t arcstat_dnode_size;
957+
aggsum_t arcstat_dnode_size;
958958
wmsum_t arcstat_bonus_size;
959959
wmsum_t arcstat_l2_hits;
960960
wmsum_t arcstat_l2_misses;

‎module/os/linux/zfs/zfs_vfsops.c‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1176,6 +1176,63 @@ zfs_root(zfsvfs_t *zfsvfs, struct inode **ipp)
11761176
return (error);
11771177
}
11781178

1179+
/*
1180+
* Dentry and inode caches referenced by a task in non-root memcg are
1181+
* not going to be scanned by the kernel-provided shrinker. So, if
1182+
* kernel prunes nothing, fall back to this manual walk to free dnodes.
1183+
* To avoid scanning the same znodes multiple times they are always rotated
1184+
* to the end of the z_all_znodes list. New znodes are inserted at the
1185+
* end of the list so we're always scanning the oldest znodes first.
1186+
*/
1187+
static int
1188+
zfs_prune_aliases(zfsvfs_t *zfsvfs, unsigned long nr_to_scan)
1189+
{
1190+
znode_t **zp_array, *zp;
1191+
int max_array = MIN(nr_to_scan, PAGE_SIZE * 8 / sizeof (znode_t *));
1192+
int objects = 0;
1193+
int i = 0, j = 0;
1194+
1195+
zp_array = vmem_zalloc(max_array * sizeof (znode_t *), KM_SLEEP);
1196+
1197+
mutex_enter(&zfsvfs->z_znodes_lock);
1198+
while ((zp = list_head(&zfsvfs->z_all_znodes)) != NULL) {
1199+
1200+
if ((i++ > nr_to_scan) || (j >= max_array))
1201+
break;
1202+
1203+
ASSERT(list_link_active(&zp->z_link_node));
1204+
list_remove(&zfsvfs->z_all_znodes, zp);
1205+
list_insert_tail(&zfsvfs->z_all_znodes, zp);
1206+
1207+
/* Skip active znodes and .zfs entries */
1208+
if (MUTEX_HELD(&zp->z_lock) || zp->z_is_ctldir)
1209+
continue;
1210+
1211+
if (igrab(ZTOI(zp)) == NULL)
1212+
continue;
1213+
1214+
zp_array[j] = zp;
1215+
j++;
1216+
}
1217+
mutex_exit(&zfsvfs->z_znodes_lock);
1218+
1219+
for (i = 0; i < j; i++) {
1220+
zp = zp_array[i];
1221+
1222+
ASSERT3P(zp, !=, NULL);
1223+
d_prune_aliases(ZTOI(zp));
1224+
1225+
if (atomic_read(&ZTOI(zp)->i_count) == 1)
1226+
objects++;
1227+
1228+
zrele(zp);
1229+
}
1230+
1231+
vmem_free(zp_array, max_array * sizeof (znode_t *));
1232+
1233+
return (objects);
1234+
}
1235+
11791236
/*
11801237
* The ARC has requested that the filesystem drop entries from the dentry
11811238
* and inode caches. This can occur when the ARC needs to free meta data
@@ -1227,6 +1284,14 @@ zfs_prune(struct super_block *sb, unsigned long nr_to_scan, int *objects)
12271284
*objects = (*shrinker->scan_objects)(shrinker, &sc);
12281285
#endif
12291286

1287+
/*
1288+
* Fall back to zfs_prune_aliases if kernel's shrinker did nothing
1289+
* due to dentry and inode caches being referenced by a task running
1290+
* in non-root memcg.
1291+
*/
1292+
if (*objects == 0)
1293+
*objects = zfs_prune_aliases(zfsvfs, nr_to_scan);
1294+
12301295
zfs_exit(zfsvfs, FTAG);
12311296

12321297
dprintf_ds(zfsvfs->z_os->os_dsl_dataset,

‎module/zfs/arc.c‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2631,7 +2631,7 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
26312631
ARCSTAT_INCR(arcstat_bonus_size, space);
26322632
break;
26332633
case ARC_SPACE_DNODE:
2634-
ARCSTAT_INCR(arcstat_dnode_size, space);
2634+
aggsum_add(&arc_sums.arcstat_dnode_size, space);
26352635
break;
26362636
case ARC_SPACE_DBUF:
26372637
ARCSTAT_INCR(arcstat_dbuf_size, space);
@@ -2677,7 +2677,7 @@ arc_space_return(uint64_t space, arc_space_type_t type)
26772677
ARCSTAT_INCR(arcstat_bonus_size, -space);
26782678
break;
26792679
case ARC_SPACE_DNODE:
2680-
ARCSTAT_INCR(arcstat_dnode_size, -space);
2680+
aggsum_add(&arc_sums.arcstat_dnode_size, -space);
26812681
break;
26822682
case ARC_SPACE_DBUF:
26832683
ARCSTAT_INCR(arcstat_dbuf_size, -space);
@@ -4490,7 +4490,7 @@ arc_evict(void)
44904490
* target is not evictable or if they go over arc_dnode_limit.
44914491
*/
44924492
int64_t prune = 0;
4493-
int64_t dn = wmsum_value(&arc_sums.arcstat_dnode_size);
4493+
int64_t dn = aggsum_value(&arc_sums.arcstat_dnode_size);
44944494
int64_t nem = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA])
44954495
+ zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA])
44964496
- zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA])
@@ -5082,11 +5082,13 @@ arc_is_overflowing(boolean_t lax, boolean_t use_reserve)
50825082
* in the ARC. In practice, that's in the tens of MB, which is low
50835083
* enough to be safe.
50845084
*/
5085-
int64_t over = aggsum_lower_bound(&arc_sums.arcstat_size) - arc_c -
5085+
int64_t arc_over = aggsum_lower_bound(&arc_sums.arcstat_size) - arc_c -
50865086
zfs_max_recordsize;
5087+
int64_t dn_over = aggsum_lower_bound(&arc_sums.arcstat_dnode_size) -
5088+
arc_dnode_limit;
50875089

50885090
/* Always allow at least one block of overflow. */
5089-
if (over < 0)
5091+
if (arc_over < 0 && dn_over <= 0)
50905092
return (ARC_OVF_NONE);
50915093

50925094
/* If we are under memory pressure, report severe overflow. */
@@ -5097,7 +5099,7 @@ arc_is_overflowing(boolean_t lax, boolean_t use_reserve)
50975099
int64_t overflow = (arc_c >> zfs_arc_overflow_shift) / 2;
50985100
if (use_reserve)
50995101
overflow *= 3;
5100-
return (over < overflow ? ARC_OVF_SOME : ARC_OVF_SEVERE);
5102+
return (arc_over < overflow ? ARC_OVF_SOME : ARC_OVF_SEVERE);
51015103
}
51025104

51035105
static abd_t *
@@ -7324,7 +7326,7 @@ arc_kstat_update(kstat_t *ksp, int rw)
73247326
#if defined(COMPAT_FREEBSD11)
73257327
as->arcstat_other_size.value.ui64 =
73267328
wmsum_value(&arc_sums.arcstat_bonus_size) +
7327-
wmsum_value(&arc_sums.arcstat_dnode_size) +
7329+
aggsum_value(&arc_sums.arcstat_dnode_size) +
73287330
wmsum_value(&arc_sums.arcstat_dbuf_size);
73297331
#endif
73307332

@@ -7366,7 +7368,7 @@ arc_kstat_update(kstat_t *ksp, int rw)
73667368
&as->arcstat_uncached_evictable_metadata);
73677369

73687370
as->arcstat_dnode_size.value.ui64 =
7369-
wmsum_value(&arc_sums.arcstat_dnode_size);
7371+
aggsum_value(&arc_sums.arcstat_dnode_size);
73707372
as->arcstat_bonus_size.value.ui64 =
73717373
wmsum_value(&arc_sums.arcstat_bonus_size);
73727374
as->arcstat_l2_hits.value.ui64 =
@@ -7736,7 +7738,7 @@ arc_state_init(void)
77367738
wmsum_init(&arc_sums.arcstat_data_size, 0);
77377739
wmsum_init(&arc_sums.arcstat_metadata_size, 0);
77387740
wmsum_init(&arc_sums.arcstat_dbuf_size, 0);
7739-
wmsum_init(&arc_sums.arcstat_dnode_size, 0);
7741+
aggsum_init(&arc_sums.arcstat_dnode_size, 0);
77407742
wmsum_init(&arc_sums.arcstat_bonus_size, 0);
77417743
wmsum_init(&arc_sums.arcstat_l2_hits, 0);
77427744
wmsum_init(&arc_sums.arcstat_l2_misses, 0);
@@ -7895,7 +7897,7 @@ arc_state_fini(void)
78957897
wmsum_fini(&arc_sums.arcstat_data_size);
78967898
wmsum_fini(&arc_sums.arcstat_metadata_size);
78977899
wmsum_fini(&arc_sums.arcstat_dbuf_size);
7898-
wmsum_fini(&arc_sums.arcstat_dnode_size);
7900+
aggsum_fini(&arc_sums.arcstat_dnode_size);
78997901
wmsum_fini(&arc_sums.arcstat_bonus_size);
79007902
wmsum_fini(&arc_sums.arcstat_l2_hits);
79017903
wmsum_fini(&arc_sums.arcstat_l2_misses);

0 commit comments

Comments
 (0)