Suspend 2 merge: 40/51: Prepare image

Nigel Cunningham
2004-11-24T16:48:16Z

This file contains the main routines used to prepare an image. Note that
this is potentially an iterative process: our allocation of metadata for
the image we know about may change the characteristics of the image and
require the allocation of a few extra pages. The number of iterations is
definitely bound (and the user can always press escape - if they've
enabled it - to cancel if I was wrong here).

We account for every page of memory. They are either:

- LRU -> pageset 2
- allocated for memory pool -> pageset 1
- otherwise used -> pageset 1
- unused and not allocated for memory pool -> not saved
- used but marked NoSave -> not saved

Plugins tell us how much memory they need, and we put that much plus a
little more in the memory pool. (Can't account for device drivers).

diff -ruN 830-prepare-image-old/kernel/power/prepare_image.c 830-prepare-image-new/kernel/power/prepare_image.c
--- 830-prepare-image-old/kernel/power/prepare_image.c 1970-01-01 10:00:00.000000000 +1000
+++ 830-prepare-image-new/kernel/power/prepare_image.c 2004-11-18 11:52:47.000000000 +1100
@@ -0,0 +1,1050 @@
+/*
+ * kernel/power/prepare_image.c
+ *
+ * Copyright (C) 2003-2004 Nigel Cunningham <ncunningham@linuxmail.org>
+ *
+ * This file is released under the GPLv2.
+ *
+ * We need to eat memory until we can:
+ * 1. Perform the save without changing anything (RAM_NEEDED < max_mapnr)
+ * 2. Fit it all in available space (active_writer->available_space() >= STORAGE_NEEDED)
+ * 3. Reload the pagedir and pageset1 to places that don't collide with their
+ * final destinations, not knowing to what extent the resumed kernel will
+ * overlap with the one loaded at boot time. I think the resumed kernel should overlap
+ * completely, but I don't want to rely on this as it is an unproven assumption. We
+ * therefore assume there will be no overlap at all (worse case).
+ * 4. Meet the user's requested limit (if any) on the size of the image.
+ * The limit is in MB, so pages/256 (assuming 4K pages).
+ *
+ * (Final test in save_image doesn't use EATEN_ENOUGH_MEMORY)
+ */
+
+#define SUSPEND_PREPARE_IMAGE_C
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/suspend.h>
+#include <linux/highmem.h>
+#include <linux/notifier.h>
+
+#include "suspend.h"
+#include "pageflags.h"
+#include "plugins.h"
+#include "proc.h"
+
+extern int pageset1_sizelow, pageset2_sizelow;
+extern unsigned long orig_mem_free;
+extern void mark_pages_for_pageset2(void);
+extern int image_size_limit;
+extern int fill_suspend_memory_pool(int sizesought);
+
+int suspend_amount_grabbed = 0;
+static int arefrozen = 0, numnosave = 0;
+static int header_space_allocated = 0;
+extern unsigned long forced_ps1_size, forced_ps2_size;
+
+/*
+ * generate_free_page_map
+ *
+ * Description: This routine generates a bitmap of free pages from the
+ * lists used by the memory manager. We then use the bitmap
+ * to quickly calculate which pages to save and in which
+ * pagesets.
+ */
+static void generate_free_page_map(void)
+{
+ int i, order, loop, cpu;
+ struct page * page;
+ unsigned long flags;
+ struct zone *zone;
+ struct per_cpu_pageset *pset;
+
+ for(i=0; i < max_mapnr; i++)
+ SetPageInUse(mem_map+i);
+
+ for_each_zone(zone) {
+ if (!zone->present_pages)
+ continue;
+ spin_lock_irqsave(&zone->lock, flags);
+ for (order = MAX_ORDER - 1; order >= 0; --order) {
+ list_for_each_entry(page, &zone->free_area[order].free_list, lru)
+ for(loop=0; loop < (1 << order); loop++) {
+ ClearPageInUse(page+loop);
+ ClearPagePageset2(page+loop);
+ }
+ }
+
+
+ for (cpu = 0; cpu < NR_CPUS; cpu++) {
+ if (!cpu_possible(cpu))
+ continue;
+
+ pset = &zone->pageset[cpu];
+
+ for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) {
+ struct per_cpu_pages *pcp;
+ struct page * page;
+
+ pcp = &pset->pcp[i];
+ list_for_each_entry(page, &pcp->list, lru) {
+ ClearPageInUse(page);
+ ClearPagePageset2(page);
+ }
+ }
+ }
+
+ spin_unlock_irqrestore(&zone->lock, flags);
+ }
+}
+
+/* size_of_free_region
+ *
+ * Description: Return the number of pages that are free, beginning with and
+ * including this one.
+ */
+static int size_of_free_region(struct page * page)
+{
+ struct page * posn = page;
+
+ while (((posn-mem_map) < max_mapnr) && (!PageInUse(posn)))
+ posn++;
+ return (posn - page);
+}
+
+static void display_reserved_pages(void)
+{
+ int loop;
+ int rangemin = -1;
+
+ for (loop = 0; loop < max_mapnr; loop++) {
+ if (PageReserved(mem_map+loop)) {
+ if (rangemin == -1)
+ rangemin = loop;
+ } else {
+ if (rangemin > -1) {
+ printk("Reserved pages from %p to %p.\n",
+ page_address(mem_map+rangemin),
+ ((char *) page_address(mem_map + loop)) - 1);
+ rangemin = -1;
+ }
+ }
+ }
+
+ if (rangemin > -1)
+ printk("Reserved pages from %p to %p.\n",
+ page_address(mem_map+rangemin),
+ ((char *) page_address(mem_map + max_mapnr)) - 1);
+}
+
+/*
+ * Description: Display which pages are marked Nosave.
+ */
+void display_nosave_pages(void)
+{
+ int loop;
+ int rangemin = -1;
+
+ if (!TEST_DEBUG_STATE(SUSPEND_NOSAVE))
+ return;
+
+ display_reserved_pages();
+
+ for (loop = 0; loop < max_mapnr; loop++) {
+ if (PageNosave(mem_map+loop)) {
+ if (rangemin == -1)
+ rangemin = loop;
+ } else {
+ if (rangemin > -1) {
+ printk("Nosave pages from %p to %p.\n",
+ page_address(mem_map+rangemin),
+ ((char *) page_address(mem_map + loop)) - 1);
+ rangemin = -1;
+ }
+ }
+ }
+
+ if (rangemin > -1)
+ printk("Nosave pages from %p to %p.\n",
+ page_address(mem_map+rangemin),
+ ((char *) page_address(mem_map + max_mapnr)) - 1);
+}
+
+/*
+ * count_data_pages
+ *
+ * This routine generates our lists of pages to be stored in each
+ * pageset. Since we store the data using ranges, and adding new
+ * ranges might allocate a new range page, this routine may well
+ * be called more than once.
+ */
+static struct pageset_sizes_result count_data_pages(void)
+{
+ int chunk_size, loop, numfree = 0;
+ int ranges = 0, currentrange = 0;
+ int usepagedir2;
+ int rangemin = 0;
+ struct pageset_sizes_result result;
+ struct range * rangepointer;
+ unsigned long value;
+#ifdef CONFIG_HIGHMEM
+ unsigned long highstart_pfn = get_highstart_pfn();
+#endif
+
+ result.size1 = 0;
+ result.size1low = 0;
+ result.size2 = 0;
+ result.size2low = 0;
+ result.needmorespace = 0;
+
+ numnosave = 0;
+
+ put_range_chain(&pagedir1.origranges);
+ put_range_chain(&pagedir1.destranges);
+ put_range_chain(&pagedir2.origranges);
+ pagedir2.destranges.first = NULL;
+ pagedir2.destranges.size = 0;
+
+ generate_free_page_map();
+
+ if (TEST_RESULT_STATE(SUSPEND_ABORTED)) {
+ result.size1 = -1;
+ result.size1low = -1;
+ result.size2 = -1;
+ result.size2low = -1;
+ result.needmorespace = 0;
+ return result;
+ }
+
+ if (max_mapnr != num_physpages) {
+ abort_suspend("Max_mapnr is not equal to num_physpages.");
+ result.size1 = -1;
+ result.size1low = -1;
+ result.size2 = -1;
+ result.size2low = -1;
+ result.needmorespace = 0;
+ return result;
+ }
+ /*
+ * Pages not to be saved are marked Nosave irrespective of being reserved
+ */
+ for (loop = 0; loop < max_mapnr; loop++) {
+ if (PageNosave(mem_map+loop)) {
+ numnosave++;
+ if (currentrange) {
+ append_to_range_chain(currentrange, rangemin, loop - 1);
+ rangemin = loop;
+ currentrange = 0;
+ }
+ continue;
+ }
+
+ if (!PageReserved(mem_map+loop)) {
+ if ((chunk_size=size_of_free_region(mem_map+loop))!=0) {
+ if (currentrange) {
+ append_to_range_chain(currentrange, rangemin, loop - 1);
+ rangemin = loop;
+ currentrange = 0;
+ }
+ numfree += chunk_size;
+ loop += chunk_size - 1;
+ continue;
+ }
+ } else {
+#ifdef CONFIG_HIGHMEM
+ if (loop >= highstart_pfn) {
+ /* HighMem pages may be marked Reserved. We ignore them. */
+ numnosave++;
+ if (currentrange) {
+ append_to_range_chain(currentrange, rangemin, loop - 1);
+ rangemin = loop;
+ currentrange = 0;
+ }
+ continue;
+ }
+#endif
+ };
+
+ usepagedir2 = !!PagePageset2(mem_map+loop);
+
+ if (currentrange != (1 + usepagedir2)) {
+ if (currentrange)
+ append_to_range_chain(currentrange, rangemin, loop - 1);
+ currentrange = usepagedir2 + 1;
+ rangemin = loop;
+ ranges++;
+ }
+
+ if (usepagedir2) {
+ result.size2++;
+ if (!PageHighMem(mem_map+loop))
+ result.size2low++;
+ } else {
+ result.size1++;
+ if (!PageHighMem(mem_map+loop))
+ result.size1low++;
+ }
+ }
+
+ if (currentrange)
+ append_to_range_chain(currentrange, rangemin, loop - 1);
+
+ if ((pagedir1.pageset_size) && (result.size1 > pagedir1.pageset_size))
+ result.needmorespace = 1;
+ if ((pagedir2.pageset_size) && (result.size2 > pagedir2.pageset_size))
+ result.needmorespace = 1;
+ suspend_message(SUSPEND_RANGES, SUSPEND_MEDIUM, 0, "Counted %d ranges.\n", ranges);
+ pagedir2.destranges.first = pagedir2.origranges.first;
+ pagedir2.destranges.size = pagedir2.origranges.size;
+ range_for_each(&pagedir1.allocdranges, rangepointer, value) {
+ add_to_range_chain(&pagedir1.destranges, value);
+ }
+
+ suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_MEDIUM, 0,
+ "Count data pages: Set1 (%d) + Set2 (%d) + Nosave (%d) + NumFree (%d) = %d.\n",
+ result.size1, result.size2, numnosave, numfree,
+ result.size1 + result.size2 + numnosave + numfree);
+ return result;
+}
+
+/* amount_needed
+ *
+ * Calculates the amount by which the image size needs to be reduced to meet
+ * our constraints.
+ */
+static int amount_needed(int use_image_size_limit)
+{
+
+ int max1 = max( (int) (RAM_TO_SUSPEND - real_nr_free_pages() -
+ nr_free_highpages() - suspend_amount_grabbed),
+ ((int) (STORAGE_NEEDED(1) -
+ active_writer->ops.writer.storage_available())));
+ if (use_image_size_limit)
+ return max( max1,
+ (image_size_limit > 0) ?
+ ((int) (STORAGE_NEEDED(1) - (image_size_limit << 8))) : 0);
+ return max1;
+}
+
+#define EATEN_ENOUGH_MEMORY() (amount_needed(1) < 1)
+unsigned long storage_available = 0;
+
+/* display_stats
+ *
+ * Display the vital statistics.
+ */
+#ifdef CONFIG_SOFTWARE_SUSPEND_DEBUG
+static void display_stats(void)
+{
+ unsigned long storage_allocated = active_writer->ops.writer.storage_allocated();
+ suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_MEDIUM, 1,
+ "Free:%d+%d+%d=%d(%d). Sets:%d(%d),%d(%d). Header:%d. Nosave:%d-%d-%d=%d. Storage:%d/%lu(%lu). Needed:%d|%d|%d.\n",
+
+ /* Free */
+ real_nr_free_pages(), suspend_amount_grabbed, suspend_memory_pool_level(0),
+ real_nr_free_pages() + suspend_amount_grabbed + suspend_memory_pool_level(0),
+ real_nr_free_pages() - nr_free_highpages(),
+
+ /* Sets */
+ pageset1_size, pageset1_sizelow,
+ pageset2_size, pageset2_sizelow,
+
+ /* Header */
+ num_range_pages,
+
+ /* Nosave */
+ numnosave, pagedir1.allocdranges.size, suspend_amount_grabbed,
+ numnosave - pagedir1.allocdranges.size - suspend_amount_grabbed,
+
+ /* Storage - converted to pages for comparison */
+ storage_allocated,
+ STORAGE_NEEDED(1),
+ storage_available,
+
+ /* Needed */
+ RAM_TO_SUSPEND - real_nr_free_pages() - nr_free_highpages() - suspend_amount_grabbed,
+ STORAGE_NEEDED(1) - storage_available,
+ (image_size_limit > 0) ? (STORAGE_NEEDED(1) - (image_size_limit << 8)) : 0);
+}
+#else
+#define display_stats() do { } while(0)
+#endif
+
+struct bloat_pages {
+ struct bloat_pages * next;
+ int order;
+};
+
+static struct bloat_pages * bloat_pages = NULL;
+
+void free_pageset_size_bloat(void)
+{
+ while (bloat_pages) {
+ struct bloat_pages * next = bloat_pages->next;
+ free_pages((unsigned long) bloat_pages, bloat_pages->order);
+ bloat_pages = next;
+ }
+}
+
+#define redo_counts() \
+{ \
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1, \
+ "Recalculating counts. Currently %ld & %ld. ", \
+ ps1_get, ps2_get); \
+ result = count_data_pages(); \
+ if (forced_ps1_size) \
+ ps1_get = forced_ps1_size - result.size1 - drop_one; \
+ if (forced_ps2_size) \
+ ps2_get = forced_ps2_size - result.size2; \
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1, \
+ "Now %ld and %ld.\n", ps1_get, ps2_get); \
+}
+
+void increase_pageset_size(struct pageset_sizes_result result)
+{
+ long ps1_get = 0, ps2_get = 0, order, j;
+ int drop_one = 0;
+
+ if (forced_ps1_size)
+ ps1_get = forced_ps1_size - result.size1;
+
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_HIGH, 1,
+ "1: Forced size = %ld. Have %d -> ps1_get = %ld.\n",
+ forced_ps1_size, result.size1, ps1_get);
+
+ /*
+ * We can make ps2 size exactly what was requested, but
+ * not both.
+ */
+ if (forced_ps2_size) {
+ ps2_get = forced_ps2_size - result.size2;
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_HIGH, 1,
+ "2: Forced size = %ld. Have %d -> ps2_get = %ld.\n",
+ forced_ps2_size, result.size2, ps2_get);
+
+ if (ps2_get > 0) {
+ order = generic_fls(ps2_get);
+ if (order >= MAX_ORDER)
+ order = MAX_ORDER - 1;
+
+ while(ps2_get > 0) {
+ struct page * newpage;
+ unsigned long virt;
+ struct bloat_pages * link;
+
+ if ((ps1_get - (1 << order)) < (1 << order))
+ redo_counts();
+
+ while ((1 << order) > (ps2_get))
+ order--;
+
+ virt = get_grabbed_pages(order);
+
+ while ((!virt) && (order > 0)) {
+ order--;
+ if ((ps1_get - (1 << order)) < (1 << order))
+ redo_counts();
+ virt = get_grabbed_pages(order);
+ }
+
+ if (!virt) {
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_MEDIUM, 1,
+ " Failed to allocate enough memory for"
+ " requested pageset sizes.\n");
+ return;
+ }
+
+ newpage = virt_to_page(virt);
+ for (j = 0; j < (1 << order); j++)
+ SetPagePageset2(newpage + j);
+
+ link = (struct bloat_pages *) virt;
+ link->next = bloat_pages;
+ link->order = order;
+ bloat_pages = link;
+
+ ps2_get -= (1 << order);
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1,
+ "Allocated %d for ps2. To get %ld.\n",
+ 1 << order, ps2_get);
+ }
+ } else
+ {
+ /* Here, we're making ps2 pages into ps1 pages */
+ int i;
+
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_HIGH, 1,
+ "Moving %ld ps2 pages to ps1.\n", -ps2_get);
+ for (i = 0; i < max_mapnr; i++) {
+ if PagePageset2(mem_map + i) {
+ ClearPagePageset2(mem_map + i);
+ ps2_get++;
+ ps1_get--;
+ }
+ if (!ps2_get)
+ break;
+ }
+ }
+ } else {
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_HIGH, 1,
+ "2: Forced size = %ld. Have %d -> ps2_get = %ld.\n",
+ forced_ps2_size, result.size2, ps2_get);
+ }
+
+ if (ps1_get > 0) {
+
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_HIGH, 1,
+ "Still to get %ld pages for ps1.\n", ps1_get);
+
+ /* We might allocate an extra range page later. */
+ if (ps1_get > 1) {
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1,
+ "Reducing ps1_get by one.\n");
+ drop_one = 1;
+ ps1_get--;
+ }
+
+ order = generic_fls(ps1_get);
+ if (order >= MAX_ORDER)
+ order = MAX_ORDER - 1;
+
+ while(ps1_get > 0) {
+ unsigned long virt;
+ struct bloat_pages * link;
+
+ if ((ps1_get - (1 << order)) < (1 << order))
+ redo_counts();
+
+ while ((1 << order) > (ps1_get))
+ order--;
+
+ virt = get_grabbed_pages(order);
+
+ while ((!virt) && (order > 0)) {
+ order--;
+ if ((ps1_get - (1 << order)) < (1 << order))
+ redo_counts();
+ virt = get_grabbed_pages(order);
+ }
+
+ if (!virt) {
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1,
+ "Couldn't get enough pages. Need %ld more.\n",
+ ps1_get);
+ return;
+ }
+
+ link = (struct bloat_pages *) virt;
+ link->next = bloat_pages;
+ link->order = order;
+ bloat_pages = link;
+
+ ps1_get -= (1 << order);
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1,
+ "Allocated %d for ps1. To get %ld.\n", 1 << order, ps1_get);
+ }
+ }
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1,
+ "Exiting increase pageset size.\n\n");
+}
+
+/*
+ * Eaten is the number of pages which have been eaten.
+ * Pagedirincluded is the number of pages which have been allocated for the pagedir.
+ */
+extern int allocate_extra_pagedir_memory(struct pagedir * p, int pageset_size, int alloc_from);
+
+struct pageset_sizes_result recalculate_stats(void)
+{
+ struct pageset_sizes_result result;
+
+ mark_pages_for_pageset2(); /* Need to call this before getting pageset1_size! */
+ result = count_data_pages();
+ suspend_message(SUSPEND_PAGESETS, SUSPEND_VERBOSE, 1,
+ "Forced sizes %ld and %ld. Result %d and %d.\n",
+ forced_ps1_size, forced_ps2_size,
+ result.size1, result.size2);
+ if ((forced_ps1_size && forced_ps1_size != result.size1) ||
+ (forced_ps2_size && forced_ps2_size != result.size2)) {
+ increase_pageset_size(result);
+ result = count_data_pages();
+ }
+ pageset1_sizelow = result.size1low;
+ pageset2_sizelow = result.size2low;
+ pagedir1.lastpageset_size = pageset1_size = result.size1;
+ pagedir2.lastpageset_size = pageset2_size = result.size2;
+ storage_available = active_writer->ops.writer.storage_available();
+ suspend_store_free_mem(SUSPEND_FREE_RANGE_PAGES, 0);
+ return result;
+}
+
+/* update_image
+ *
+ * Allocate [more] memory and storage for the image.
+ * Remember, this is iterative!
+ */
+static int update_image(void)
+{
+ struct pageset_sizes_result result;
+ int iteration = 0, orig_num_range_pages;
+
+ result = recalculate_stats();
+
+ suspend_store_free_mem(SUSPEND_FREE_RANGE_PAGES, 0);
+
+ do {
+ iteration++;
+
+ orig_num_range_pages = num_range_pages;
+
+ suspend_message(SUSPEND_ANY_SECTION, SUSPEND_LOW, 1,
+ "-- Iteration %d.\n", iteration);
+
+ if (suspend_allocate_checksum_pages()) {
+ suspend_message(SUSPEND_ANY_SECTION, SUSPEND_LOW, 1,
+ "Still need to get more pages for checksum pages.\n");
+ return 1;
+ }
+
+ if (allocate_extra_pagedir_memory(&pagedir1, pageset1_size, pageset2_sizelow)) {
+ suspend_message(SUSPEND_ANY_SECTION, SUSPEND_LOW, 1,
+ "Still need to get more pages for pagedir 1.\n");
+ return 1;
+ }
+
+ if (active_writer->ops.writer.allocate_storage(MAIN_STORAGE_NEEDED(1))) {
+ suspend_message(SUSPEND_ANY_SECTION, SUSPEND_LOW, 1,
+ "Still need to get more storage space for the image proper.\n");
+ suspend_store_free_mem(SUSPEND_FREE_WRITER_STORAGE, 0);
+ return 1;
+ }
+
+ suspend_store_free_mem(SUSPEND_FREE_WRITER_STORAGE, 0);
+
+ set_suspend_state(SUSPEND_SLAB_ALLOC_FALLBACK);
+
+ if (active_writer->ops.writer.allocate_header_space(HEADER_STORAGE_NEEDED)) {
+ suspend_message(SUSPEND_ANY_SECTION, SUSPEND_LOW, 1,
+ "Still need to get more storage space for header.\n");
+ return 1;
+ }
+
+ header_space_allocated = HEADER_STORAGE_NEEDED;
+
+ clear_suspend_state(SUSPEND_SLAB_ALLOC_FALLBACK);
+
+ /*
+ * Allocate remaining storage space, if possible, up to the
+ * maximum we know we'll need. It's okay to allocate the
+ * maximum if the writer is the swapwriter, but
+ * we don't want to grab all available space on an NFS share.
+ * We therefore ignore the expected compression ratio here,
+ * thereby trying to allocate the maximum image size we could
+ * need (assuming compression doesn't expand the image), but
+ * don't complain if we can't get the full amount we're after.
+ */
+
+ active_writer->ops.writer.allocate_storage(
+ max((long)(active_writer->ops.writer.storage_available() -
+ active_writer->ops.writer.storage_allocated()),
+ (long)(HEADER_STORAGE_NEEDED + MAIN_STORAGE_NEEDED(1))));
+
+ suspend_store_free_mem(SUSPEND_FREE_WRITER_STORAGE, 0);
+
+ result = recalculate_stats();
+ display_stats();
+
+ } while (((orig_num_range_pages < num_range_pages) ||
+ result.needmorespace ||
+ header_space_allocated < HEADER_STORAGE_NEEDED ||
+ active_writer->ops.writer.storage_allocated() < (HEADER_STORAGE_NEEDED + MAIN_STORAGE_NEEDED(1)))
+ && (!TEST_RESULT_STATE(SUSPEND_ABORTED)));
+
+ suspend_message(SUSPEND_ANY_SECTION, SUSPEND_MEDIUM, 1, "-- Exit loop.\n");
+
+ return (amount_needed(0) > 0);
+}
+
+/* ----------------------- Memory grabbing --------------------------
+ *
+ * All of the memory that is available, we grab.
+ * This enables us to get the image size down, even when other
+ * processes might be trying to increase their memory usage. (We
+ * have a hook to disable the OOM killer).
+ *
+ * At the same time, suspend's own routines get memory from this
+ * pool, and so does slab growth. Only get_zeroed_page and siblings
+ * see no memory available.
+ */
+
+static spinlock_t suspend_grabbed_memory_lock = SPIN_LOCK_UNLOCKED;
+
+struct eaten_memory_t
+{
+ void * next;
+};
+
+struct eaten_memory_t *eaten_memory[MAX_ORDER];
+
+static void __grab_free_memory(void)
+{
+ int order, k;
+
+ /*
+ * First, quickly eat all memory that's already free.
+ */
+
+ for (order = MAX_ORDER - 1; order > -1; order--) {
+ struct eaten_memory_t *prev = eaten_memory[order];
+ eaten_memory[order] = (struct eaten_memory_t *) __get_free_pages(GFP_ATOMIC, order);
+ while (eaten_memory[order]) {
+ struct page * page = virt_to_page(eaten_memory[order]);
+ eaten_memory[order]->next = prev;
+ prev = eaten_memory[order];
+ suspend_amount_grabbed += (1 << order);
+ for (k=0; k < (1 << order); k++) {
+ SetPageNosave(page + k);
+ ClearPagePageset2(page + k);
+ }
+ eaten_memory[order] = (struct eaten_memory_t *) __get_free_pages(GFP_ATOMIC, order);
+ }
+ eaten_memory[order] = prev;
+ }
+}
+
+static void grab_free_memory(void)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&suspend_grabbed_memory_lock, flags);
+ __grab_free_memory();
+ spin_unlock_irqrestore(&suspend_grabbed_memory_lock, flags);
+}
+
+static void free_grabbed_memory(void)
+{
+ struct eaten_memory_t *next = NULL, *this = NULL;
+ int j, num_freed = 0, order;
+ unsigned long flags;
+
+ spin_lock_irqsave(&suspend_grabbed_memory_lock, flags);
+
+ /* Free all eaten pages immediately */
+ for (order = MAX_ORDER - 1; order > -1; order--) {
+ this=eaten_memory[order];
+ while(this) {
+ struct page * page = virt_to_page(this);
+ next = this->next;
+ for (j=0; j < (1 << order); j++)
+ ClearPageNosave(page + j);
+ free_pages((unsigned long) this, order);
+ num_freed+= (1 << order);
+ this = next;
+ }
+ eaten_memory[order] = NULL;
+ }
+ suspend_amount_grabbed -= num_freed;
+ BUG_ON(suspend_amount_grabbed);
+ spin_unlock_irqrestore(&suspend_grabbed_memory_lock, flags);
+}
+
+unsigned long get_grabbed_pages(int order)
+{
+ unsigned long this = (unsigned long) eaten_memory[order];
+ int alternative, j;
+ unsigned long flags;
+ struct page * page;
+
+ /* Get grabbed lowmem pages for suspend's use */
+ spin_lock_irqsave(&suspend_grabbed_memory_lock, flags);
+
+try_again:
+ if (this) {
+ page = virt_to_page(this);
+ eaten_memory[order] = eaten_memory[order]->next;
+ for (j=0; j < (1 << order); j++) {
+ ClearPageNosave(page + j);
+ ClearPagePageset2(page + j);
+ clear_page(page_address(page + j));
+ }
+ suspend_amount_grabbed -= (1 << order);
+ spin_unlock_irqrestore(&suspend_grabbed_memory_lock, flags);
+ return this;
+ }
+
+ alternative = order+1;
+ while ((!eaten_memory[alternative]) && (alternative < MAX_ORDER))
+ alternative++;
+
+ /* Maybe we didn't eat any memory - try normal get */
+ if (alternative == MAX_ORDER) {
+ this = __get_free_pages(GFP_ATOMIC, order);
+ if (this) {
+ page = virt_to_page(this);
+ for (j=0; j < (1 << order); j++) {
+ clear_page((char *) this + j * PAGE_SIZE);
+ ClearPagePageset2(page + j);
+ }
+ }
+ spin_unlock_irqrestore(&suspend_grabbed_memory_lock, flags);
+ return this;
+ }
+
+ {
+ unsigned long virt = (unsigned long) eaten_memory[alternative];
+ page = virt_to_page(eaten_memory[alternative]);
+ eaten_memory[alternative] = eaten_memory[alternative]->next;
+ for (j=0; j < (1 << (alternative)); j++) {
+ ClearPageNosave(page + j);
+ clear_page(page_address(page + j));
+ ClearPagePageset2(page + j);
+ }
+ free_pages(virt, alternative);
+ suspend_amount_grabbed -= (1 << alternative);
+ }
+
+ /* Get the chunk we want to return. May fail if something grabs
+ * the memory before us. */
+ this = __get_free_pages(GFP_ATOMIC, order);
+ if (!this)
+ goto try_again;
+
+ page = virt_to_page(this);
+
+ /* Grab the rest */
+ __grab_free_memory();
+
+ spin_unlock_irqrestore(&suspend_grabbed_memory_lock, flags);
+
+ return this;
+}
+
+/* --------------------------------------------------------------------------- */
+
+extern int freeze_processes(int no_progress);
+
+static int attempt_to_freeze(void)
+{
+ int result;
+
+ /* Stop processes before checking again */
+ thaw_processes(FREEZER_ALL_THREADS);
+ prepare_status(1, 1, "Freezing processes");
+ result = freeze_processes(0);
+ suspend_message(SUSPEND_FREEZER, SUSPEND_VERBOSE, 0, "- Freeze_processes returned %d.\n",
+ result);
+
+ if (result) {
+ SET_RESULT_STATE(SUSPEND_ABORTED);
+ SET_RESULT_STATE(SUSPEND_FREEZING_FAILED);
+ } else
+ arefrozen = 1;
+
+ return result;
+}
+
+extern asmlinkage long sys_sync(void);
+
+static int eat_memory(void)
+{
+ int orig_memory_still_to_eat, last_amount_needed = 0, times_criteria_met = 0;
+ int free_flags = 0, did_eat_memory = 0;
+
+ /*
+ * Note that if we have enough storage space and enough free memory, we may
+ * exit without eating anything. We give up when the last 10 iterations ate
+ * no extra pages because we're not going to get much more anyway, but
+ * the few pages we get will take a lot of time.
+ *
+ * We freeze processes before beginning, and then unfreeze them if we
+ * need to eat memory until we think we have enough. If our attempts
+ * to freeze fail, we give up and abort.
+ */
+
+ /* ----------- Stage 1: Freeze Processes ------------- */
+
+
+ prepare_status(0, 1, "Eating memory.");
+
+ recalculate_stats();
+ display_stats();
+
+ orig_memory_still_to_eat = amount_needed(1);
+ last_amount_needed = orig_memory_still_to_eat;
+
+ switch (image_size_limit) {
+ case -1: /* Don't eat any memory */
+ if (orig_memory_still_to_eat) {
+ SET_RESULT_STATE(SUSPEND_ABORTED);
+ SET_RESULT_STATE(SUSPEND_WOULD_EAT_MEMORY);
+ }
+ break;
+ case -2: /* Free caches only */
+ free_flags = GFP_NOIO | __GFP_HIGHMEM;
+ break;
+ default:
+ free_flags = GFP_ATOMIC | __GFP_HIGHMEM;
+ }
+
+ /* ----------- Stage 2: Eat memory ------------- */
+
+ while (((!EATEN_ENOUGH_MEMORY()) || (image_size_limit == -2)) && (!TEST_RESULT_STATE(SUSPEND_ABORTED)) && (times_criteria_met < 10)) {
+ int amount_freed;
+ int amount_wanted = orig_memory_still_to_eat - amount_needed(1);
+ if (amount_wanted < 1)
+ amount_wanted = 1; /* image_size_limit == -2 */
+
+ suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_VERBOSE, 1,
+ "Times met criteria is %d.\n", times_criteria_met);
+ if (orig_memory_still_to_eat)
+ update_status(orig_memory_still_to_eat - amount_needed(1), orig_memory_still_to_eat, " Image size %d ", MB(STORAGE_NEEDED(1)));
+ else
+ update_status(0, 1, "Image size %d ", MB(STORAGE_NEEDED(1)));
+
+ if ((last_amount_needed - amount_needed(1)) < 10)
+ times_criteria_met++;
+ else
+ times_criteria_met = 0;
+ last_amount_needed = amount_needed(1);
+ amount_freed = shrink_all_memory(last_amount_needed);
+ suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_VERBOSE, 1,
+ "Given %d, shrink_all_memory returned %d.\n", last_amount_needed, amount_freed);
+ grab_free_memory();
+ recalculate_stats();
+ display_stats();
+
+ did_eat_memory = 1;
+
+ check_shift_keys(0, NULL);
+ }
+
+ grab_free_memory();
+
+ suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_VERBOSE, 1,
+ "Out of main eat memory loop.\n");
+
+ if (did_eat_memory) {
+ unsigned long orig_state = get_suspend_state();
+ suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_VERBOSE, 1,
+ "Ate memory; letting kjournald etc run.\n");
+ clear_suspend_state(SUSPEND_USE_MEMORY_POOL);
+ thaw_processes(FREEZER_KERNEL_THREADS);
+ /* Freeze_processes will call sys_sync too */
+ freeze_processes(1);
+ grab_free_memory();
+ restore_suspend_state(orig_state);
+ recalculate_stats();
+ display_stats();
+ }
+
+ suspend_message(SUSPEND_EAT_MEMORY, 1, SUSPEND_VERBOSE, "\n");
+
+ suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_VERBOSE, 1,
+ "(Freezer exit:) Swap needed calculated as (%d+%d)*%d/100+%d+1+%d=%d.\n",
+ pageset1_size,
+ pageset2_size,
+ expected_compression_ratio(),
+ num_range_pages,
+ HEADER_STORAGE_NEEDED,
+ STORAGE_NEEDED(1));
+
+ /* Blank out image size display */
+ update_status(100, 100, " ");
+
+ /* Include image size limit when checking what to report */
+ if (amount_needed(1) > 0)
+ SET_RESULT_STATE(SUSPEND_UNABLE_TO_FREE_ENOUGH_MEMORY);
+
+ /* But don't include it when deciding whether to abort (soft limit) */
+ if ((amount_needed(0) > 0)) {
+ printk("Unable to free sufficient memory to suspend. Still need %d pages. "
+ "You may be able to avoid this problem by reducing the async_io_limit\n",
+ amount_needed(1));
+ SET_RESULT_STATE(SUSPEND_ABORTED);
+ }
+
+ check_shift_keys(1, "Memory eating completed.");
+ return 0;
+}
+
+/* prepare_image
+ *
+ * Entry point to the whole image preparation section.
+ *
+ * We do four things:
+ * - Freeze processes;
+ * - Ensure image size constraints are met;
+ * - Complete all the preparation for saving the image,
+ * including allocation of storage. The only memory
+ * that should be needed when we're finished is that
+ * for actually storing the image (and we know how
+ * much is needed for that because the plugins tell
+ * us).
+ * - Make sure that all dirty buffers are written out.
+ */
+int prepare_image(void)
+{
+ int result = 1, sizesought;
+
+ arefrozen = 0;
+
+ header_space_allocated = 0;
+
+ sizesought = 100 + memory_for_plugins();
+
+ PRINTFREEMEM("prior to filling the memory pool");
+
+ if (fill_suspend_memory_pool(sizesought))
+ return 1;
+
+ PRINTFREEMEM("after filling the memory pool");
+ suspend_store_free_mem(SUSPEND_FREE_MEM_POOL, 0);
+
+ if (attempt_to_freeze())
+ return 1;
+
+ PRINTFREEMEM("after freezing processes");
+ suspend_store_free_mem(SUSPEND_FREE_FREEZER, 0);
+
+ if (!active_writer->ops.writer.storage_available()) {
+ printk(KERN_ERR "You need some storage available to be able to suspend.\n");
+ SET_RESULT_STATE(SUSPEND_ABORTED);
+ SET_RESULT_STATE(SUSPEND_NOSTORAGE_AVAILABLE);
+ return 1;
+ }
+
+ do {
+ if (eat_memory() || TEST_RESULT_STATE(SUSPEND_ABORTED))
+ break;
+
+ PRINTFREEMEM("after eating memory");
+ suspend_store_free_mem(SUSPEND_FREE_EAT_MEMORY, 0);
+
+ /* Top up */
+ if (fill_suspend_memory_pool(sizesought))
+ continue;
+
+ PRINTFREEMEM("after refilling memory pool");
+ suspend_store_free_mem(SUSPEND_FREE_MEM_POOL, 0);
+
+ result = update_image();
+ PRINTFREEMEM("after updating the image");
+
+ } while ((result) && (!TEST_RESULT_STATE(SUSPEND_ABORTED)) &&
+ (!TEST_RESULT_STATE(SUSPEND_UNABLE_TO_FREE_ENOUGH_MEMORY)));
+
+ PRINTFREEMEM("after preparing image");
+
+ /* Release memory that has been eaten */
+ free_grabbed_memory();
+
+ PRINTFREEMEM("after freeing grabbed memory");
+ suspend_store_free_mem(SUSPEND_FREE_GRABBED_MEMORY, 1);
+
+ set_suspend_state(SUSPEND_USE_MEMORY_POOL);
+
+ check_shift_keys(1, "Image preparation complete.");
+
+ return result;
+}
+
+EXPORT_SYMBOL(suspend_amount_grabbed);