5ab365dd41
Problem:
We received a report of a workload that causes an xfs task to be blocked
for more than 120 seconds on log reservation via iomap_ioend completion
batching.
kernel: err [5636141.631454] INFO: task xfs-conv/dm-4:1788 blocked for
more than 122 seconds.
kernel: info [267022.728862] Workqueue: xfs-conv/dm-4 xfs_end_io [xfs]
kernel: info [267022.728864] Call Trace:
kernel: info [267022.728870] __schedule+0x340/0x810
kernel: info [267022.728876] schedule+0x51/0xc0
kernel: info [267022.728913] xlog_grant_head_wait+0xc7/0x200 [xfs]
kernel: info [267022.728950] xlog_grant_head_check+0xd0/0x110 [xfs]
kernel: info [267022.728985] xfs_log_reserve+0xc3/0x1e0 [xfs]
kernel: info [267022.729023] xfs_trans_reserve+0x156/0x1b0 [xfs]
kernel: info [267022.729184] xfs_trans_alloc+0xc6/0x190 [xfs]
kernel: info [267022.729317] xfs_iomap_write_unwritten+0xaa/0x2c0 [xfs]
kernel: info [267022.729333] ? stop_one_cpu+0x71/0xa0
kernel: info [267022.729347] ? set_cpus_allowed_ptr+0x10/0x10
kernel: info [267022.729396] xfs_end_ioend+0xc4/0x100 [xfs]
kernel: info [267022.729444] ? xfs_setfilesize_ioend+0x60/0x60 [xfs]
kernel: info [267022.729491] xfs_end_io+0xb9/0xe0 [xfs]
kernel: info [267022.729505] process_one_work+0x1a1/0x370
kernel: info [267022.729516] rescuer_thread+0x207/0x350
kernel: info [267022.729528] ? worker_thread+0x370/0x370
kernel: info [267022.729537] kthread+0x12e/0x150
kernel: info [267022.729548] ? __kthread_cancel_work+0x40/0x40
kernel: info [267022.729559] ret_from_fork+0x1f/0x30
After that, the connection via ssh to the controller is stuck,
Press Ctrl+C, it entered shell and the prompt displayed '-sh-4.2$'
Solution:
Removing the preallocated transaction from xfs append ioends to avoid
the ioend completion batching log reservation deadlock.
Now we continue to process the append ioend completions via the
workqueue, but let the wq task allocate the transaction similar to other
ioend types.
Backport the four patches from upstream(git://git.kernel.org/pub/scm/
linux/kernel/git/torvalds/linux.git) for debian-based StarlingX.
Only the 0034-xfs-use-current-journal_info-for-detecting-transacti.patch
for centos-based StarlingX is from stable tree(git://git.kernel.org/pub/
scm/linux/kernel/git/stable/linux.git linux-5.10.y branch), because the
kernel has been upgraded to v5.10.152 for debian-based StarlingX which
includes this fix, so we just apply it for the centos-based one.
TestPlan:
Pass: Execute bonnie++ test for xfs filesystem successfully without
kernel panic and any xfs anomalies in the kernel logs.
$mkfs.x /dev/sdc1
$mount /dev/sdc1 ~/xfstests
$sudo bonnie++ -u root:root -d ~/xfstests
Debian:
Pass: build-pkgs -c -a
Pass: build-image
Pass: boot successfully with std/rt.
CentOS:
Pass: build-pkgs
Pass: build-iso
Pass: boot successfully with std/rt.
Closes-Bug: 1996269
Signed-off-by: Zhixiong Chi <zhixiong.chi@windriver.com>
Change-Id: I1e5b85111b2b54cd249c116724b952042f9d781f
95 lines
2.7 KiB
Diff
95 lines
2.7 KiB
Diff
From 170e31793806ce5e5a9647b6340954536244518e Mon Sep 17 00:00:00 2001
|
|
From: Brian Foster <bfoster@redhat.com>
|
|
Date: Fri, 9 Apr 2021 10:27:55 -0700
|
|
Subject: [PATCH 3/4] xfs: drop unused ioend private merge and setfilesize code
|
|
|
|
XFS no longer attaches anthing to ioend->io_private. Remove the
|
|
unnecessary ->io_private merging code. This removes the only remaining
|
|
user of xfs_setfilesize_ioend() so remove that function as well.
|
|
|
|
Signed-off-by: Brian Foster <bfoster@redhat.com>
|
|
Reviewed-by: Christoph Hellwig <hch@lst.de>
|
|
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
|
|
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
|
|
[commit 044c6449f18f174ba8d86640936add3fc7582e49 upstream
|
|
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git]
|
|
Signed-off-by: Zhixiong Chi <zhixiong.chi@windriver.com>
|
|
---
|
|
fs/xfs/xfs_aops.c | 46 +---------------------------------------------
|
|
1 file changed, 1 insertion(+), 45 deletions(-)
|
|
|
|
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
|
|
index a74a9ead5..23a9bd9ff 100644
|
|
--- a/fs/xfs/xfs_aops.c
|
|
+++ b/fs/xfs/xfs_aops.c
|
|
@@ -85,31 +85,6 @@ xfs_setfilesize(
|
|
return __xfs_setfilesize(ip, tp, offset, size);
|
|
}
|
|
|
|
-STATIC int
|
|
-xfs_setfilesize_ioend(
|
|
- struct iomap_ioend *ioend,
|
|
- int error)
|
|
-{
|
|
- struct xfs_inode *ip = XFS_I(ioend->io_inode);
|
|
- struct xfs_trans *tp = ioend->io_private;
|
|
-
|
|
- /*
|
|
- * The transaction may have been allocated in the I/O submission thread,
|
|
- * thus we need to mark ourselves as being in a transaction manually.
|
|
- * Similarly for freeze protection.
|
|
- */
|
|
- xfs_trans_set_context(tp);
|
|
- __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS);
|
|
-
|
|
- /* we abort the update if there was an IO error */
|
|
- if (error) {
|
|
- xfs_trans_cancel(tp);
|
|
- return error;
|
|
- }
|
|
-
|
|
- return __xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size);
|
|
-}
|
|
-
|
|
/*
|
|
* IO write completion.
|
|
*/
|
|
@@ -163,25 +138,6 @@ xfs_end_ioend(
|
|
memalloc_nofs_restore(nofs_flag);
|
|
}
|
|
|
|
-/*
|
|
- * If the to be merged ioend has a preallocated transaction for file
|
|
- * size updates we need to ensure the ioend it is merged into also
|
|
- * has one. If it already has one we can simply cancel the transaction
|
|
- * as it is guaranteed to be clean.
|
|
- */
|
|
-static void
|
|
-xfs_ioend_merge_private(
|
|
- struct iomap_ioend *ioend,
|
|
- struct iomap_ioend *next)
|
|
-{
|
|
- if (!ioend->io_private) {
|
|
- ioend->io_private = next->io_private;
|
|
- next->io_private = NULL;
|
|
- } else {
|
|
- xfs_setfilesize_ioend(next, -ECANCELED);
|
|
- }
|
|
-}
|
|
-
|
|
/* Finish all pending io completions. */
|
|
void
|
|
xfs_end_io(
|
|
@@ -201,7 +157,7 @@ xfs_end_io(
|
|
while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
|
|
io_list))) {
|
|
list_del_init(&ioend->io_list);
|
|
- iomap_ioend_try_merge(ioend, &tmp, xfs_ioend_merge_private);
|
|
+ iomap_ioend_try_merge(ioend, &tmp, NULL);
|
|
xfs_end_ioend(ioend);
|
|
}
|
|
}
|
|
--
|
|
2.34.1
|
|
|