From 0066b862bb3a09f39295abd5d972a53ac8dc1555 Mon Sep 17 00:00:00 2001
From: Peter Bergin <peter@berginkonsult.se>
Date: Wed, 19 Sep 2018 15:12:31 +0200
Subject: [PATCH] rpm/rpmio.c: restrict virtual memory usage if limit set

A solution to avoid OOM situation when the virtual memory is restricted
for a user (ulimit -v). As the lzopen_internal function is run in parallel
one instance per CPU thread the available virtual memory is limited per
CPU thread.

Upstream-Status: Pending [merge of multithreading patches to upstream]

Signed-off-by: Peter Bergin <peter@berginkonsult.se>
---
 rpmio/rpmio.c | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c
index e051c98..b3c56b6 100644
--- a/rpmio/rpmio.c
+++ b/rpmio/rpmio.c
@@ -845,6 +845,40 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
 		}
 #endif
 
+		struct rlimit virtual_memory;
+		getrlimit(RLIMIT_AS, &virtual_memory);
+		if (virtual_memory.rlim_cur != RLIM_INFINITY) {
+			const uint64_t virtual_memlimit = virtual_memory.rlim_cur;
+			const uint64_t virtual_memlimit_per_cpu_thread =
+				virtual_memlimit / lzma_cputhreads();
+			uint64_t memory_usage_virt;
+			rpmlog(RPMLOG_NOTICE, "XZ: virtual memory restricted to %lu and "
+			       "per CPU thread %lu\n", virtual_memlimit, virtual_memlimit_per_cpu_thread);
+			/* keep reducing the number of compression threads until memory
+			   usage falls below the limit per CPU thread*/
+			while ((memory_usage_virt = lzma_stream_encoder_mt_memusage(&mt_options)) >
+			       virtual_memlimit_per_cpu_thread) {
+				/* If number of threads goes down to zero lzma_stream_encoder will
+				 * will return UINT64_MAX. We must check here to avoid an infinite loop.
+				 * If we get into situation that one thread requires more virtual memory
+				 * than available we set one thread, print error message and try anyway. */
+				if (--mt_options.threads == 0) {
+					mt_options.threads = 1;
+					rpmlog(RPMLOG_WARNING,
+					       "XZ: Could not adjust number of threads to get below "
+					       "virtual memory limit %lu. usage %lu\n",
+					       virtual_memlimit_per_cpu_thread, memory_usage_virt);
+					break;
+				}
+			}
+			if (threads != (int)mt_options.threads)
+				rpmlog(RPMLOG_NOTICE,
+				       "XZ: Adjusted the number of threads from %d to %d to not "
+				       "exceed the memory usage limit of %lu bytes\n",
+				       threads, mt_options.threads, virtual_memlimit);
+
+		}
+
 		ret = lzma_stream_encoder_mt(&lzfile->strm, &mt_options);
 	    }
 #endif
-- 
2.7.4