1 /* 2 * Generic hugetlb support. 3 * (C) William Irwin, April 2004 4 */ 5 #include <linux/gfp.h> 6 #include <linux/list.h> 7 #include <linux/init.h> 8 #include <linux/module.h> 9 #include <linux/mm.h> 10 #include <linux/hugetlb.h> 11 #include <linux/sysctl.h> 12 #include <linux/highmem.h> 13 14 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL; 15 static unsigned long nr_huge_pages, free_huge_pages; 16 unsigned long max_huge_pages; 17 static struct list_head hugepage_freelists[MAX_NUMNODES]; 18 static unsigned int nr_huge_pages_node[MAX_NUMNODES]; 19 static unsigned int free_huge_pages_node[MAX_NUMNODES]; 20 static spinlock_t hugetlb_lock = SPIN_LOCK_UNLOCKED; 21 22 static void enqueue_huge_page(struct page *page) 23 { 24 int nid = page_zone(page)->zone_pgdat->node_id; 25 list_add(&page->lru, &hugepage_freelists[nid]); 26 free_huge_pages++; 27 free_huge_pages_node[nid]++; 28 } 29 30 static struct page *dequeue_huge_page(void) 31 { 32 int nid = numa_node_id(); 33 struct page *page = NULL; 34 35 if (list_empty(&hugepage_freelists[nid])) { 36 for (nid = 0; nid < MAX_NUMNODES; ++nid) 37 if (!list_empty(&hugepage_freelists[nid])) 38 break; 39 } 40 if (nid >= 0 && nid < MAX_NUMNODES && 41 !list_empty(&hugepage_freelists[nid])) { 42 page = list_entry(hugepage_freelists[nid].next, 43 struct page, lru); 44 list_del(&page->lru); 45 free_huge_pages--; 46 free_huge_pages_node[nid]--; 47 } 48 return page; 49 } 50 51 static struct page *alloc_fresh_huge_page(void) 52 { 53 static int nid = 0; 54 struct page *page; 55 page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP, 56 HUGETLB_PAGE_ORDER); 57 nid = (nid + 1) % numnodes; 58 if (page) { 59 nr_huge_pages++; 60 nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]++; 61 } 62 return page; 63 } 64 65 void free_huge_page(struct page *page) 66 { 67 BUG_ON(page_count(page)); 68 69 INIT_LIST_HEAD(&page->lru); 70 page[1].mapping = NULL; 71 72 spin_lock(&hugetlb_lock); 73 enqueue_huge_page(page); 74 spin_unlock(&hugetlb_lock); 75 } 76 77 struct page *alloc_huge_page(void) 78 { 79 struct page *page; 80 int i; 81 82 spin_lock(&hugetlb_lock); 83 page = dequeue_huge_page(); 84 if (!page) { 85 spin_unlock(&hugetlb_lock); 86 return NULL; 87 } 88 spin_unlock(&hugetlb_lock); 89 set_page_count(page, 1); 90 page[1].mapping = (void *)free_huge_page; 91 for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i) 92 clear_highpage(&page[i]); 93 return page; 94 } 95 96 static int __init hugetlb_init(void) 97 { 98 unsigned long i; 99 struct page *page; 100 101 for (i = 0; i < MAX_NUMNODES; ++i) 102 INIT_LIST_HEAD(&hugepage_freelists[i]); 103 104 for (i = 0; i < max_huge_pages; ++i) { 105 page = alloc_fresh_huge_page(); 106 if (!page) 107 break; 108 spin_lock(&hugetlb_lock); 109 enqueue_huge_page(page); 110 spin_unlock(&hugetlb_lock); 111 } 112 max_huge_pages = free_huge_pages = nr_huge_pages = i; 113 printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages); 114 return 0; 115 } 116 module_init(hugetlb_init); 117 118 static int __init hugetlb_setup(char *s) 119 { 120 if (sscanf(s, "%lu", &max_huge_pages) <= 0) 121 max_huge_pages = 0; 122 return 1; 123 } 124 __setup("hugepages=", hugetlb_setup); 125 126 static void update_and_free_page(struct page *page) 127 { 128 int i; 129 nr_huge_pages--; 130 nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--; 131 for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) { 132 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | 133 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved | 134 1 << PG_private | 1<< PG_writeback); 135 set_page_count(&page[i], 0); 136 } 137 set_page_count(page, 1); 138 __free_pages(page, HUGETLB_PAGE_ORDER); 139 } 140 141 #ifdef CONFIG_HIGHMEM 142 static void try_to_free_low(unsigned long count) 143 { 144 int i, nid; 145 for (i = 0; i < MAX_NUMNODES; ++i) { 146 struct page *page, *next; 147 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) { 148 if (PageHighMem(page)) 149 continue; 150 list_del(&page->lru); 151 update_and_free_page(page); 152 nid = page_zone(page)->zone_pgdat->node_id; 153 free_huge_pages--; 154 free_huge_pages_node[nid]--; 155 if (count >= nr_huge_pages) 156 return; 157 } 158 } 159 } 160 #else 161 static inline void try_to_free_low(unsigned long count) 162 { 163 } 164 #endif 165 166 static unsigned long set_max_huge_pages(unsigned long count) 167 { 168 while (count > nr_huge_pages) { 169 struct page *page = alloc_fresh_huge_page(); 170 if (!page) 171 return nr_huge_pages; 172 spin_lock(&hugetlb_lock); 173 enqueue_huge_page(page); 174 spin_unlock(&hugetlb_lock); 175 } 176 if (count >= nr_huge_pages) 177 return nr_huge_pages; 178 179 spin_lock(&hugetlb_lock); 180 try_to_free_low(count); 181 while (count < nr_huge_pages) { 182 struct page *page = dequeue_huge_page(); 183 if (!page) 184 break; 185 update_and_free_page(page); 186 } 187 spin_unlock(&hugetlb_lock); 188 return nr_huge_pages; 189 } 190 191 #ifdef CONFIG_SYSCTL 192 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 193 struct file *file, void __user *buffer, 194 size_t *length, loff_t *ppos) 195 { 196 proc_doulongvec_minmax(table, write, file, buffer, length, ppos); 197 max_huge_pages = set_max_huge_pages(max_huge_pages); 198 return 0; 199 } 200 #endif /* CONFIG_SYSCTL */ 201 202 int hugetlb_report_meminfo(char *buf) 203 { 204 return sprintf(buf, 205 "HugePages_Total: %5lu\n" 206 "HugePages_Free: %5lu\n" 207 "Hugepagesize: %5lu kB\n", 208 nr_huge_pages, 209 free_huge_pages, 210 HPAGE_SIZE/1024); 211 } 212 213 int hugetlb_report_node_meminfo(int nid, char *buf) 214 { 215 return sprintf(buf, 216 "Node %d HugePages_Total: %5u\n" 217 "Node %d HugePages_Free: %5u\n", 218 nid, nr_huge_pages_node[nid], 219 nid, free_huge_pages_node[nid]); 220 } 221 222 int is_hugepage_mem_enough(size_t size) 223 { 224 return (size + ~HPAGE_MASK)/HPAGE_SIZE <= free_huge_pages; 225 } 226 227 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 228 unsigned long hugetlb_total_pages(void) 229 { 230 return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE); 231 } 232 EXPORT_SYMBOL(hugetlb_total_pages); 233 234 /* 235 * We cannot handle pagefaults against hugetlb pages at all. They cause 236 * handle_mm_fault() to try to instantiate regular-sized pages in the 237 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 238 * this far. 239 */ 240 static struct page *hugetlb_nopage(struct vm_area_struct *vma, 241 unsigned long address, int *unused) 242 { 243 BUG(); 244 return NULL; 245 } 246 247 struct vm_operations_struct hugetlb_vm_ops = { 248 .nopage = hugetlb_nopage, 249 }; 250 251 void zap_hugepage_range(struct vm_area_struct *vma, 252 unsigned long start, unsigned long length) 253 { 254 struct mm_struct *mm = vma->vm_mm; 255 256 spin_lock(&mm->page_table_lock); 257 unmap_hugepage_range(vma, start, start + length); 258 spin_unlock(&mm->page_table_lock); 259 } 260
This page was automatically generated by LXR 0.3.1. • Linux is a registered trademark of Linus Torvalds