Don't dump pages which only contain zero bytes

Introduces a new command line option '--skip-zero-bytes'
which detects pages which only contain zero bytes and
prohibits that they get dumped in the processes image file.
It is a potentially expensive operation because it checks for
every single process page if it contains only zeros, but
it can significantly decrease the image size and improve the
startup-time if many such pages exist. It effectively
replaces such pages which the kernel's zero-page on restore.

Signed-off-by: Volker Simonis <[email protected]>

Author: simonis

Documentation/criu.txt

@@ -369,6 +369,14 @@ mount -t cgroup -o devices,freezer none devices,freezer
     Deduplicate "old" data in pages images of previous *dump*. This option
     implies incremental *dump* mode (see the *pre-dump* command).
 
+*--skip-zero-pages*::
+    Don't dump pages containing only zero bytes. This is a
+    potentially expensive operation because it checks for
+    every single process page if it contains only zeros, but
+    it can significantly decrease the image size and improve the
+    startup-time if many such pages exist. It effectively
+    replaces such pages which the kernel's zero-page on restore.
+
 *-l*, *--file-locks*::
     Dump file locks. It is necessary to make sure that all file lock users
     are taken into dump, so it is only safe to use this for enclosed containers

criu/config.c

@@ -650,6 +650,7 @@ int parse_options(int argc, char **argv, bool *usage_error, bool *has_exec_cmd,
 		{ "ms", no_argument, 0, 1054 },
 		BOOL_OPT("track-mem", &opts.track_mem),
 		BOOL_OPT("auto-dedup", &opts.auto_dedup),
+		BOOL_OPT("skip-zero-pages", &opts.skip_zero_pages),
 		{ "libdir", required_argument, 0, 'L' },
 		{ "cpu-cap", optional_argument, 0, 1057 },
 		BOOL_OPT("force-irmap", &opts.force_irmap),

criu/cr-service.c

@@ -541,6 +541,9 @@ static int setup_opts_from_req(int sk, CriuOpts *req)
 	if (req->has_auto_dedup)
 		opts.auto_dedup = req->auto_dedup;
 
+	if (req->has_skip_zero_pages)
+		opts.skip_zero_pages = req->skip_zero_pages;
+
 	if (req->has_force_irmap)
 		opts.force_irmap = req->force_irmap;
 

criu/crtools.c

@@ -541,6 +541,7 @@ int main(int argc, char *argv[], char *envp[])
 	       "                        pages images of previous dump\n"
 	       "                        when used on restore, as soon as page is restored, it\n"
 	       "                        will be punched from the image\n"
+	       "  --skip-zero-pages     don't dump pages containing only zero bytes.\n"
 	       "  --pre-dump-mode       splice - parasite based pre-dumping (default)\n"
 	       "                        read   - process_vm_readv syscall based pre-dumping\n"
 	       "\n"

criu/include/cr_options.h

@@ -157,6 +157,7 @@ struct cr_options {
 	int track_mem;
 	char *img_parent;
 	int auto_dedup;
+	int skip_zero_pages;
 	unsigned int cpu_cap;
 	int force_irmap;
 	char **exec_cmd;

criu/include/stats.h

@@ -33,6 +33,8 @@ enum {
 	CNT_SHPAGES_SKIPPED_PARENT,
 	CNT_SHPAGES_WRITTEN,
 
+	CNT_SKIPPED_ZERO_PAGES,
+
 	DUMP_CNT_NR_STATS,
 };
 

criu/mem.c

@@ -3,8 +3,10 @@
 #include <sys/mman.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <string.h>
 #include <sys/syscall.h>
 #include <sys/prctl.h>
+#include <sys/uio.h>
 
 #include "types.h"
 #include "cr_options.h"
@@ -31,6 +33,7 @@
 #include "prctl.h"
 #include "compel/infect-util.h"
 #include "pidfd-store.h"
+#include "xmalloc.h"
 
 #include "protobuf.h"
 #include "images/pagemap.pb-c.h"
@@ -191,11 +194,33 @@ static int generate_iovs(struct pstree_item *item, struct vma_area *vma, struct
 			 bool has_parent)
 {
 	unsigned long nr_scanned;
-	unsigned long pages[3] = {};
+	/* Counters for PAGES_SKIPPED_PARENT, PAGES_LAZY, PAGES_WRITTEN and SKIPPED_ZERO_PAGES */
+	unsigned long pages[4] = {};
 	unsigned long vaddr;
 	bool dump_all_pages;
 	int ret = 0;
 
+	static char *zero_page = NULL;
+	static char *remote_page = NULL;
+	int zero = 0;
+	struct iovec local[2];
+	struct iovec remote[1];
+	int nread = 0;
+	if (opts.skip_zero_pages && zero_page == NULL) {
+		zero_page = xmalloc(PAGE_SIZE);
+		remote_page = xmalloc(PAGE_SIZE);
+		if (zero_page == NULL || remote_page == NULL) {
+			pr_warn("Can't allocate memory - disabling --skip-zero-pages\n");
+			opts.skip_zero_pages = 0;
+		} else {
+			memzero(zero_page, PAGE_SIZE);
+			local[0].iov_base = remote_page;
+			local[0].iov_len = PAGE_SIZE;
+			remote[0].iov_base = (void *)0x0;
+			remote[0].iov_len = PAGE_SIZE;
+		}
+	}
+
 	dump_all_pages = should_dump_entire_vma(vma->e);
 
 	nr_scanned = 0;
@@ -207,9 +232,25 @@ static int generate_iovs(struct pstree_item *item, struct vma_area *vma, struct
 
 		/* If dump_all_pages is true, should_dump_page is called to get pme. */
 		next = should_dump_page(pmc, vma->e, vaddr, &softdirty);
-		if (!dump_all_pages && next != vaddr) {
-			vaddr = next - PAGE_SIZE;
-			continue;
+		if (!dump_all_pages) {
+			if (next != vaddr) {
+				vaddr = next - PAGE_SIZE;
+				continue;
+			} else if (opts.skip_zero_pages) {
+				remote[0].iov_base = (void *)vaddr;
+				nread = process_vm_readv(item->pid->real, local, 1, remote, 1, 0);
+				if (nread == PAGE_SIZE) {
+					zero = memcmp(zero_page, remote_page, PAGE_SIZE);
+					/*
+                                         * If the page contains just zeros we can treat it like the zero page and skip it.
+                                         * At restore it will be replaced by a reference to the zero page and COWed if accessed.
+                                         */
+					if (zero == 0) {
+						pages[3]++;
+						continue;
+					}
+				}
+			}
 		}
 
 		if (vma_entry_can_be_lazy(vma->e) && !is_stack(item, vaddr))
@@ -247,8 +288,10 @@ static int generate_iovs(struct pstree_item *item, struct vma_area *vma, struct
 	cnt_add(CNT_PAGES_SKIPPED_PARENT, pages[0]);
 	cnt_add(CNT_PAGES_LAZY, pages[1]);
 	cnt_add(CNT_PAGES_WRITTEN, pages[2]);
+	cnt_add(CNT_SKIPPED_ZERO_PAGES, pages[3]);
 
-	pr_info("Pagemap generated: %lu pages (%lu lazy) %lu holes\n", pages[2] + pages[1], pages[1], pages[0]);
+	pr_info("Pagemap generated: %lu pages (%lu lazy) %lu holes %lu skipped zero\n",
+		pages[2] + pages[1], pages[1], pages[0], pages[3]);
 	return ret;
 }
 

criu/stats.c

@@ -134,6 +134,9 @@ static void display_stats(int what, StatsEntry *stats)
 		       stats->dump->pages_skipped_parent, stats->dump->pages_skipped_parent);
 		pr_msg("Memory pages written: %" PRIu64 " (0x%" PRIx64 ")\n", stats->dump->pages_written,
 		       stats->dump->pages_written);
+		if (stats->dump->has_skipped_zero_pages)
+			pr_msg("Memory pages skipped because zero: %" PRIu64 " (0x%" PRIx64 ")\n",
+			       stats->dump->skipped_zero_pages, stats->dump->skipped_zero_pages);
 		pr_msg("Lazy memory pages: %" PRIu64 " (0x%" PRIx64 ")\n", stats->dump->pages_lazy,
 		       stats->dump->pages_lazy);
 	} else if (what == RESTORE_STATS) {
@@ -178,6 +181,10 @@ void write_stats(int what)
 		ds_entry.has_page_pipes = true;
 		ds_entry.page_pipe_bufs = dstats->counts[CNT_PAGE_PIPE_BUFS];
 		ds_entry.has_page_pipe_bufs = true;
+		if (opts.skip_zero_pages) {
+			ds_entry.has_skipped_zero_pages = true;
+			ds_entry.skipped_zero_pages = dstats->counts[CNT_SKIPPED_ZERO_PAGES];
+		}
 
 		ds_entry.shpages_scanned = dstats->counts[CNT_SHPAGES_SCANNED];
 		ds_entry.has_shpages_scanned = true;

images/rpc.proto

@@ -145,6 +145,7 @@ message criu_opts {
 	optional bool			leave_stopped		= 69;
 	optional bool			display_stats		= 70;
 	optional bool			log_to_stderr		= 71;
+	optional bool			skip_zero_pages		= 72;
 /*	optional bool			check_mounts		= 128;	*/
 }
 

images/stats.proto

@@ -22,6 +22,8 @@ message dump_stats_entry {
 	optional uint64			shpages_scanned		= 12;
 	optional uint64			shpages_skipped_parent	= 13;
 	optional uint64			shpages_written		= 14;
+
+	optional uint64			skipped_zero_pages	= 15;
 }
 
 message restore_stats_entry {