aboutsummaryrefslogtreecommitdiff
path: root/mm/page_alloc.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/page_alloc.c')
-rw-r--r--mm/page_alloc.c78
1 files changed, 60 insertions, 18 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a6bd1d8d2f5..b249d6ce348 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -56,6 +56,7 @@
#include <linux/ftrace_event.h>
#include <linux/memcontrol.h>
#include <linux/prefetch.h>
+#include <linux/pasr.h>
#include <linux/migrate.h>
#include <linux/page-debug-flags.h>
#include <linux/hugetlb.h>
@@ -373,9 +374,11 @@ void prep_compound_page(struct page *page, unsigned long order)
__SetPageHead(page);
for (i = 1; i < nr_pages; i++) {
struct page *p = page + i;
- __SetPageTail(p);
set_page_count(p, 0);
p->first_page = page;
+ /* Make sure p->first_page is always valid for PageTail() */
+ smp_wmb();
+ __SetPageTail(p);
}
}
@@ -583,6 +586,7 @@ static inline void __free_one_page(struct page *page,
} else {
list_del(&buddy->lru);
zone->free_area[order].nr_free--;
+ pasr_kget(buddy, order);
rmv_page_order(buddy);
}
combined_idx = buddy_idx & page_idx;
@@ -616,6 +620,7 @@ static inline void __free_one_page(struct page *page,
list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
out:
zone->free_area[order].nr_free++;
+ pasr_kput(page, order);
}
static inline int free_pages_check(struct page *page)
@@ -850,6 +855,7 @@ static inline void expand(struct zone *zone, struct page *page,
#endif
list_add(&page[size].lru, &area->free_list[migratetype]);
area->nr_free++;
+ pasr_kput(page, high);
set_page_order(&page[size], high);
}
}
@@ -918,6 +924,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
list_del(&page->lru);
rmv_page_order(page);
area->nr_free--;
+ pasr_kget(page, current_order);
expand(zone, page, order, current_order, area, migratetype);
set_freepage_migratetype(page, migratetype);
return page;
@@ -935,7 +942,7 @@ static int fallbacks[MIGRATE_TYPES][4] = {
[MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
#ifdef CONFIG_CMA
- [MIGRATE_MOVABLE] = { MIGRATE_CMA, MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
+ [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
[MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */
#else
[MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
@@ -1043,7 +1050,8 @@ static void change_pageblock_range(struct page *pageblock_page,
* as well.
*/
static void try_to_steal_freepages(struct zone *zone, struct page *page,
- int start_type, int fallback_type)
+ int start_type, int fallback_type,
+ int start_order)
{
int current_order = page_order(page);
@@ -1053,9 +1061,14 @@ static void try_to_steal_freepages(struct zone *zone, struct page *page,
return;
}
- if (current_order >= pageblock_order / 2 ||
+ /* don't let unmovable allocations cause migrations simply because of free pages */
+ if ((start_type != MIGRATE_UNMOVABLE && current_order >= pageblock_order / 2) ||
+ /* only steal reclaimable page blocks for unmovable allocations */
+ (start_type == MIGRATE_UNMOVABLE && fallback_type != MIGRATE_MOVABLE && current_order >= pageblock_order / 2) ||
+ /* reclaimable can steal aggressively */
start_type == MIGRATE_RECLAIMABLE ||
- start_type == MIGRATE_UNMOVABLE ||
+ // allow unmovable allocs up to 64K without migrating blocks
+ (start_type == MIGRATE_UNMOVABLE && start_order >= 5) ||
page_group_by_mobility_disabled) {
int pages;
@@ -1075,11 +1088,12 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
struct free_area * area;
unsigned int current_order;
struct page *page;
+ unsigned int non_cma_order;
/* Find the largest possible block of pages in the other list */
- for (current_order = MAX_ORDER-1;
- current_order >= order && current_order <= MAX_ORDER-1;
- --current_order) {
+ for (non_cma_order = MAX_ORDER-1;
+ non_cma_order >= order && non_cma_order <= MAX_ORDER-1;
+ --non_cma_order) {
int i;
for (i = 0;; i++) {
int migratetype = fallbacks[start_migratetype][i];
@@ -1089,6 +1103,18 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
if (migratetype == MIGRATE_RESERVE)
break;
+ if (is_migrate_cma(migratetype))
+ /* CMA page blocks are not movable across
+ * migrate types. Seach for free blocks
+ * from lowest order to avoid contiguous
+ * higher alignment allocations for subsequent
+ * alloc requests.
+ */
+ current_order = order + MAX_ORDER - 1 -
+ non_cma_order;
+ else
+ current_order = non_cma_order;
+
area = &(zone->free_area[current_order]);
if (list_empty(&area->free_list[migratetype]))
continue;
@@ -1096,11 +1122,12 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
page = list_entry(area->free_list[migratetype].next,
struct page, lru);
area->nr_free--;
+ pasr_kget(page, current_order);
if (!is_migrate_cma(migratetype)) {
try_to_steal_freepages(zone, page,
- start_migratetype,
- migratetype);
+ start_migratetype,
+ migratetype, order);
} else {
/*
* When borrowing from MIGRATE_CMA, we need to
@@ -1465,6 +1492,8 @@ static int __isolate_free_page(struct page *page, unsigned int order)
/* Remove page from free list */
list_del(&page->lru);
zone->free_area[order].nr_free--;
+ pasr_kget(page, order);
+
rmv_page_order(page);
/* Set the pageblock if the isolated page is at least a pageblock */
@@ -2161,6 +2190,14 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
}
/*
+ * PM-freezer should be notified that there might be an OOM killer on
+ * its way to kill and wake somebody up. This is too early and we might
+ * end up not killing anything but false positives are acceptable.
+ * See freeze_processes.
+ */
+ note_oom_kill();
+
+ /*
* Go through the zonelist yet one more time, keep very high watermark
* here, this is only to catch a parallel oom killing, we must fail if
* we're still under heavy pressure.
@@ -2380,7 +2417,7 @@ static inline int
gfp_to_alloc_flags(gfp_t gfp_mask)
{
int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
- const gfp_t wait = gfp_mask & __GFP_WAIT;
+ const bool atomic = !(gfp_mask & (__GFP_WAIT | __GFP_NO_KSWAPD));
/* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
@@ -2389,20 +2426,20 @@ gfp_to_alloc_flags(gfp_t gfp_mask)
* The caller may dip into page reserves a bit more if the caller
* cannot run direct reclaim, or if the caller has realtime scheduling
* policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
- * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
+ * set both ALLOC_HARDER (atomic == true) and ALLOC_HIGH (__GFP_HIGH).
*/
alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
- if (!wait) {
+ if (atomic) {
/*
- * Not worth trying to allocate harder for
- * __GFP_NOMEMALLOC even if it can't schedule.
+ * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
+ * if it can't schedule.
*/
- if (!(gfp_mask & __GFP_NOMEMALLOC))
+ if (!(gfp_mask & __GFP_NOMEMALLOC))
alloc_flags |= ALLOC_HARDER;
/*
- * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
- * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
+ * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
+ * comment for __cpuset_node_allowed_softwall().
*/
alloc_flags &= ~ALLOC_CPUSET;
} else if (unlikely(rt_task(current)) && !in_interrupt())
@@ -6191,6 +6228,11 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
list_del(&page->lru);
rmv_page_order(page);
zone->free_area[order].nr_free--;
+ pasr_kget(page, order);
+#ifdef CONFIG_HIGHMEM
+ if (PageHighMem(page))
+ totalhigh_pages -= 1 << order;
+#endif
for (i = 0; i < (1 << order); i++)
SetPageReserved((page+i));
pfn += (1 << order);
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 02:28:30 +0000