Reimplement AVL trees
[monolithium.git] / kernel / src / memory / memory.c
1 /*
2  * memory.c
3  *
4  * Copyright (C) 2016 Aleksandar Andrejevic <theflash@sdf.lonestar.org>
5  *
6  * This program is free software: you can redistribute it and/or modify
7  * it under the terms of the GNU Affero General Public License as
8  * published by the Free Software Foundation, either version 3 of the
9  * License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU Affero General Public License for more details.
15  *
16  * You should have received a copy of the GNU Affero General Public License
17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19
20 #include <memory.h>
21 #include <exception.h>
22 #include <process.h>
23 #include <syscalls.h>
24 #include <heap.h>
25 #include <cpu.h>
26
27 static void **physical_memory_stack = (void**)MEM_STACK_VIRT_ADDR;
28 static lock_t phys_mem_stack_lock = 0;
29 static page_t *pages = NULL;
30 static void *current_page_directory = INVALID_PAGE;
31 static memory_address_space_t kernel_address_space;
32 static memory_address_space_t mapping_space;
33 static list_entry_t user_address_spaces = { &user_address_spaces, &user_address_spaces };
34 static dword_t total_physical_pages = 0;
35 static dword_t num_free_pages = 0;
36 static dword_t mem_tree_bitmap[TOTAL_PAGES / 32];
37 static lock_t mem_tree_lock = 0;
38 static semaphore_t temporary_page_semaphore;
39 static bool_t evicting = FALSE;
40 static DECLARE_LIST(transition_pages);
41 static DECLARE_LIST(page_stores);
42 static lock_t page_store_lock = 0;
43
44 static void *evict_page(void);
45
46 static inline void invalidate_tlb(dword_t *addr)
47 {
48     asm volatile ("invlpg %0" :: "m"(*addr));
49 }
50
51 static inline void *alloc_physical_page(void)
52 {
53     void *page = INVALID_PAGE;
54
55     if (!evicting && num_free_pages <= EVICTION_THRESHOLD)
56     {
57         evicting = TRUE;
58         page = evict_page();
59         evicting = FALSE;
60
61         if (page != INVALID_PAGE) return page;
62     }
63
64     acquire_lock(&phys_mem_stack_lock);
65     if (num_free_pages) page = physical_memory_stack[--num_free_pages];
66     release_lock(&phys_mem_stack_lock);
67
68     return page;
69 }
70
71 static inline void free_physical_page(void *address)
72 {
73     acquire_lock(&phys_mem_stack_lock);
74     physical_memory_stack[num_free_pages++] = address;
75     release_lock(&phys_mem_stack_lock);
76 }
77
78 static int compare_page(const void *a, const void *b)
79 {
80     const page_t *page_a = (const page_t*)a;
81     const page_t *page_b = (const page_t*)b;
82
83     if (page_a->phys_addr < page_b->phys_addr) return -1;
84     else if (page_a->phys_addr > page_b->phys_addr) return 1;
85     else return 0;
86 }
87
88 static page_t *get_page(void *physical)
89 {
90     page_t key = { .phys_addr = (uintptr_t)physical };
91     if (pages == NULL) return NULL;
92     return (page_t*)bsearch(&key, pages, total_physical_pages, sizeof(page_t), compare_page);
93 }
94
95 static inline dword_t reference_page(void *physical)
96 {
97     page_t *page = get_page(physical);
98     if (!page) return 0;
99
100     return ++page->ref_count;
101 }
102
103 static inline dword_t dereference_page(void *physical)
104 {
105     page_t *page = get_page(physical);
106     if (!page) return 0;
107
108     return --page->ref_count;
109 }
110
111 static dword_t map_page(void *physical, void *virtual, dword_t flags)
112 {
113     dword_t i;
114     dword_t ret = ERR_SUCCESS;
115     critical_t critical;
116     dword_t phys_addr = PAGE_ALIGN((dword_t)physical);
117     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
118     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
119     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
120     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
121
122     flags &= 0x00000FFF;
123     enter_critical(&critical);
124
125     if (!(page_directory[pd_index] & PAGE_PRESENT))
126     {
127         void *table_page = alloc_physical_page();
128         if (table_page == INVALID_PAGE)
129         {
130             ret = ERR_NOMEMORY;
131             goto done;
132         }
133
134         reference_page(table_page);
135         page_directory[pd_index] = (dword_t)table_page | PAGE_PRESENT | PAGE_WRITABLE;
136
137         invalidate_tlb(page_table);
138         for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++) page_table[i] = 0;
139     }
140
141     page_directory[pd_index] |= flags;
142     if (page_table[pt_index] & PAGE_PRESENT)
143     {
144         ret = ERR_EXISTS;
145         goto done;
146     }
147
148     reference_page((void*)phys_addr);
149     page_table[pt_index] = phys_addr | flags | PAGE_PRESENT;
150     invalidate_tlb(virtual);
151
152 done:
153     leave_critical(&critical);
154     return ret;
155 }
156
157 static dword_t unmap_page(void *virtual)
158 {
159     dword_t i, ret = ERR_SUCCESS;
160     critical_t critical;
161     bool_t empty_dir = TRUE;
162     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
163     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
164     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
165     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
166
167     enter_critical(&critical);
168
169     if (!(page_directory[pd_index] & PAGE_PRESENT))
170     {
171         ret = ERR_NOTFOUND;
172         goto done;
173     }
174
175     if (!(page_table[pt_index] & PAGE_PRESENT))
176     {
177         ret = ERR_NOTFOUND;
178         goto done;
179     }
180
181     dereference_page((void*)PAGE_ALIGN(page_table[pt_index]));
182     page_table[pt_index] = 0;
183     invalidate_tlb((dword_t*)virt_addr);
184
185     for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++) if (page_table[i])
186     {
187         empty_dir = FALSE;
188         break;
189     }
190
191     if (empty_dir)
192     {
193         void *table_page = (void*)PAGE_ALIGN(page_directory[pd_index]);
194         page_directory[pd_index] = 0;
195         invalidate_tlb(page_table);
196
197         if (dereference_page(table_page) == 0)
198         {
199             free_physical_page(table_page);
200         }
201     }
202
203 done:
204     leave_critical(&critical);
205     return ret;
206 }
207
208 static dword_t get_page_flags(void *virtual)
209 {
210     dword_t virt_addr = PAGE_ALIGN((uintptr_t)virtual);
211     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
212     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
213     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
214
215     if (!(page_directory[pd_index] & PAGE_PRESENT)) return 0;
216     if (!(page_table[pt_index] & PAGE_PRESENT)) return 0;
217
218     return PAGE_OFFSET(page_table[pt_index]);
219 }
220
221 static dword_t set_page_flags(void *virtual, dword_t flags)
222 {
223     dword_t ret = ERR_SUCCESS;
224     critical_t critical;
225     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
226     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
227     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
228     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
229
230     flags &= 0x00000FFF;
231     enter_critical(&critical);
232
233     if (!(page_directory[pd_index] & PAGE_PRESENT))
234     {
235         ret = ERR_NOTFOUND;
236         goto done;
237     }
238
239     if (!(page_table[pt_index] & PAGE_PRESENT))
240     {
241         ret = ERR_NOTFOUND;
242         goto done;
243     }
244
245     page_directory[pd_index] |= flags;
246     page_table[pt_index] = PAGE_ALIGN(page_table[pt_index]) | flags | PAGE_PRESENT;
247     invalidate_tlb((void*)virt_addr);
248
249 done:
250     leave_critical(&critical);
251     return ret;
252 }
253
254 static void *map_temporary_page(void *physical, dword_t flags)
255 {
256     int i;
257     wait_semaphore(&temporary_page_semaphore, 1, NO_TIMEOUT);
258
259     for (i = TEMPORARY_PAGES - 1; i >= temporary_page_semaphore.count ; i--)
260     {
261         void *address = (void*)(TEMPORARY_ADDR + i * PAGE_SIZE);
262
263         if (get_physical_address(address) == INVALID_PAGE)
264         {
265             if (map_page(physical, address, flags) == ERR_SUCCESS) return address;
266             break;
267         }
268     }
269
270     return NULL;
271 }
272
273 static void unmap_temporary_page(void *virtual)
274 {
275     unmap_page(virtual);
276     release_semaphore(&temporary_page_semaphore, 1);
277 }
278
279 static inline dword_t alloc_page(void *virtual, dword_t flags)
280 {
281     void *phys = alloc_physical_page();
282     if (phys == INVALID_PAGE) return ERR_NOMEMORY;
283
284     dword_t ret = map_page(phys, virtual, flags);
285     if (ret != ERR_SUCCESS) free_physical_page(phys);
286
287     return ret;
288 }
289
290 static inline dword_t free_page(void *virtual)
291 {
292     void *phys = get_physical_address(virtual);
293     if (phys == INVALID_PAGE) return ERR_INVALID;
294
295     unmap_page(virtual);
296
297     page_t *page = get_page(phys);
298     if (page == NULL || page->ref_count == 0) free_physical_page(phys);
299
300     return ERR_SUCCESS;
301 }
302
303 static void *evict_page_from_address_space(memory_address_space_t *space)
304 {
305     void *physical = INVALID_PAGE;
306     int chances = 2;
307     dword_t cached_directory[PAGE_SIZE / sizeof(dword_t)];
308     dword_t *table = NULL;
309
310     if (read_physical(space->page_directory, cached_directory, PAGE_SIZE) != ERR_SUCCESS)
311     {
312         return INVALID_PAGE;
313     }
314
315     if (!space->evict_blk_ptr) space->evict_blk_ptr = space->evictable_blocks.next;
316     memory_block_t *block = CONTAINER_OF(space->evict_blk_ptr, memory_block_t, evict_link);
317     dword_t prev_pd_index = (dword_t)-1;
318     dword_t address;
319     dword_t pd_index, pt_index;
320
321     while (chances)
322     {
323         address = (dword_t)block->address + space->evict_page_num * PAGE_SIZE;
324         pd_index = ADDR_TO_PDE(address);
325         pt_index = ADDR_TO_PTE(address);
326         if (!(cached_directory[pd_index] & PAGE_PRESENT)) goto next;
327
328         if (prev_pd_index != pd_index)
329         {
330             if (table) unmap_temporary_page(table);
331             table = map_temporary_page((void*)PAGE_ALIGN(cached_directory[pd_index]),
332                                        PAGE_PRESENT | PAGE_WRITABLE);
333             if (table == NULL) break;
334             prev_pd_index = pd_index;
335         }
336
337         if (table[pt_index])
338         {
339             if (!(table[pt_index] & PAGE_ACCESSED))
340             {
341                 physical = (void*)PAGE_ALIGN(table[pt_index]);
342                 break;
343             }
344
345             table[pt_index] &= ~PAGE_ACCESSED;
346         }
347
348 next:
349         space->evict_page_num++;
350
351         if (space->evict_page_num == (dword_t)block->size)
352         {
353             space->evict_page_num = 0;
354             space->evict_blk_ptr = space->evict_blk_ptr->next;
355
356             if (space->evict_blk_ptr == &space->evictable_blocks)
357             {
358                 space->evict_blk_ptr = space->evict_blk_ptr->next;
359                 chances--;
360             }
361
362             if (space->evict_blk_ptr == &space->evictable_blocks) break;
363             block = CONTAINER_OF(space->evict_blk_ptr, memory_block_t, evict_link);
364         }
365     }
366
367     if (physical == INVALID_PAGE) goto cleanup;
368
369     dword_t i;
370     list_entry_t *ptr;
371     page_store_t *store = NULL;
372     byte_t buffer[PAGE_SIZE];
373
374     dword_t ret = read_physical(physical, buffer, PAGE_SIZE);
375     if (ret != ERR_SUCCESS)
376     {
377         physical = INVALID_PAGE;
378         goto cleanup;
379     }
380
381     for (ptr = page_stores.next; ptr != &page_stores; ptr = ptr->next)
382     {
383         store = CONTAINER_OF(ptr, page_store_t, link);
384
385         for (i = 0; i < store->max_entries; i++) if (!test_bit(store->bitmap, i)) break;
386         if (i == store->max_entries) continue;
387     }
388
389     if (ptr == &page_stores)
390     {
391         physical = INVALID_PAGE;
392         goto cleanup;
393     }
394
395     page_store_entry_t *entry = (page_store_entry_t*)malloc(sizeof(page_store_entry_t));
396     if (entry == NULL)
397     {
398         physical = INVALID_PAGE;
399         goto cleanup;
400     }
401
402     space->stats.evicted += PAGE_SIZE;
403     entry->address = (void*)address;
404     entry->address_space = space;
405     entry->number = INVALID_STORE_NUMBER;
406     entry->physical = INVALID_PAGE;
407
408     if (dereference_page(physical) == 0)
409     {
410         entry->number = i;
411
412         dword_t bytes_written;
413         ret = syscall_write_file(store->file_handle, buffer, (qword_t)entry->number * (qword_t)PAGE_SIZE, PAGE_SIZE, &bytes_written);
414         if (ret != ERR_SUCCESS)
415         {
416             reference_page(physical);
417             free(entry);
418             physical = INVALID_PAGE;
419             goto cleanup;
420         }
421
422         set_bit(store->bitmap, i);
423         list_append(&store->entry_list, &entry->link);
424
425         for (ptr = transition_pages.next; ptr != &transition_pages; ptr = ptr->next)
426         {
427             page_store_entry_t *other_entry = CONTAINER_OF(ptr, page_store_entry_t, link);
428
429             if (other_entry->physical == physical)
430             {
431                 ASSERT(other_entry->number == INVALID_STORE_NUMBER);
432
433                 list_remove(&other_entry->link);
434                 list_append(&store->entry_list, &other_entry->link);
435
436                 other_entry->number = entry->number;
437                 other_entry->physical = INVALID_PAGE;
438             }
439         }
440     }
441     else
442     {
443         entry->physical = physical;
444         list_append(&transition_pages, &entry->link);
445         physical = INVALID_PAGE;
446     }
447
448     table[pt_index] = 0;
449     if (space->page_directory == get_page_directory()) invalidate_tlb((void*)address);
450
451 cleanup:
452     if (table) unmap_temporary_page(table);
453     return physical;
454 }
455
456 static void *evict_page(void)
457 {
458     if (pages == NULL) return INVALID_PAGE;
459
460     list_entry_t *ptr;
461
462     for (ptr = user_address_spaces.next; ptr != &user_address_spaces; ptr = ptr->next)
463     {
464         memory_address_space_t *space = CONTAINER_OF(ptr, memory_address_space_t, link);
465         void *page = evict_page_from_address_space(space);
466         if (page != INVALID_PAGE) return page;
467     }
468
469     return evict_page_from_address_space(&kernel_address_space);
470 }
471
472 static memory_block_t *mem_tree_alloc(void)
473 {
474     dword_t i;
475     memory_block_t *block = NULL;
476
477     acquire_lock(&mem_tree_lock);
478     for (i = 0; i < TOTAL_PAGES; i++) if (!test_bit(mem_tree_bitmap, i)) break;
479
480     if (i < TOTAL_PAGES)
481     {
482         block = (memory_block_t*)(MEM_TREE_BLOCKS + i * sizeof(memory_block_t));
483
484         if ((get_physical_address(block) != INVALID_PAGE)
485             || (alloc_page(block, PAGE_GLOBAL | PAGE_WRITABLE | PAGE_PRESENT) == ERR_SUCCESS))
486         {
487             set_bit(mem_tree_bitmap, i);
488         }
489         else
490         {
491             block = NULL;
492         }
493     }
494
495     release_lock(&mem_tree_lock);
496     return block;
497 }
498
499 static void mem_tree_free(memory_block_t *block)
500 {
501     dword_t index = ((dword_t)block - MEM_TREE_BLOCKS) / sizeof(memory_block_t);
502     bool_t busy = FALSE;
503     dword_t i, page = PAGE_ALIGN((dword_t)block);
504
505     acquire_lock(&mem_tree_lock);
506     clear_bit(mem_tree_bitmap, index);
507
508     for (i = page; i < page + PAGE_SIZE; i += sizeof(memory_block_t))
509     {
510         index = (i - MEM_TREE_BLOCKS) / sizeof(memory_block_t);
511         if (test_bit(mem_tree_bitmap, index))
512         {
513             busy = TRUE;
514             break;
515         }
516     }
517
518     if (!busy) free_page((void*)page);
519     release_lock(&mem_tree_lock);
520 }
521
522 static memory_block_t *find_block_by_addr_internal(memory_block_t *block, void *address)
523 {
524     qword_t key = (qword_t)(dword_t)address;
525     qword_t start_addr = block->address;
526     qword_t end_addr = start_addr + block->size * PAGE_SIZE;
527
528     if (key >= start_addr && key < end_addr) return block;
529
530     if (key < start_addr)
531     {
532         if (!block->by_addr_node.left) return NULL;
533
534         memory_block_t *left_block = CONTAINER_OF(block->by_addr_node.left, memory_block_t, by_addr_node);
535         return find_block_by_addr_internal(left_block, address);
536     }
537     else
538     {
539         if (!block->by_addr_node.right) return NULL;
540
541         memory_block_t *right_block = CONTAINER_OF(block->by_addr_node.right, memory_block_t, by_addr_node);
542         return find_block_by_addr_internal(right_block, address);
543     }
544 }
545
546 static memory_block_t *find_block_by_addr(memory_address_space_t *space, void *address)
547 {
548     if (!space->by_addr_tree.root) return NULL;
549     memory_block_t *root = CONTAINER_OF(space->by_addr_tree.root, memory_block_t, by_addr_node);
550     return find_block_by_addr_internal(root, address);
551 }
552
553 static bool_t clone_blocks_recursive(memory_address_space_t *space, memory_block_t *block)
554 {
555     memory_block_t *clone = mem_tree_alloc();
556     if (clone == NULL) return FALSE;
557
558     clone->address = block->address;
559     clone->size = block->size;
560     block->flags |= MEMORY_BLOCK_COPY_ON_WRITE;
561     clone->flags = block->flags;
562     clone->address_space = space;
563     clone->section = block->section;
564
565     avl_tree_insert(&space->by_addr_tree, &clone->by_addr_node);
566     avl_tree_insert(&space->by_size_tree, &clone->by_size_node);
567
568     memory_block_t *left_block = CONTAINER_OF(block->by_addr_node.left, memory_block_t, by_addr_node);
569     memory_block_t *right_block = CONTAINER_OF(block->by_addr_node.right, memory_block_t, by_addr_node);
570
571     if ((block->by_addr_node.left && !clone_blocks_recursive(space, left_block))
572         || (block->by_addr_node.right && !clone_blocks_recursive(space, right_block)))
573     {
574         avl_tree_remove(&space->by_addr_tree, &clone->by_addr_node);
575         avl_tree_remove(&space->by_size_tree, &clone->by_size_node);
576         mem_tree_free(clone);
577         return FALSE;
578     }
579
580     return TRUE;
581 }
582
583 static inline void release_memory_block(memory_block_t *block)
584 {
585     dword_t page;
586     dword_t start_address = (dword_t)block->address;
587     dword_t end_address = start_address + (dword_t)block->size * PAGE_SIZE;
588
589     critical_t critical;
590     enter_critical(&critical);
591     void *old_page_dir = get_page_directory();
592     set_page_directory(block->address_space->page_directory);
593
594     for (page = start_address; page < end_address; page += PAGE_SIZE)
595     {
596         free_page((void*)page);
597     }
598
599     set_page_directory(old_page_dir);
600     leave_critical(&critical);
601
602     if (block->section)
603     {
604         dereference(&block->section->header);
605         block->section = NULL;
606     }
607
608     list_entry_t *i;
609
610     for (i = transition_pages.next; i != &transition_pages; i = i->next)
611     {
612         page_store_entry_t *entry = CONTAINER_OF(i, page_store_entry_t, link);
613
614         if (entry->address_space == block->address_space
615             && (dword_t)entry->address >= start_address
616             && ((dword_t)entry->address < end_address))
617         {
618             list_remove(&entry->link);
619             free(entry);
620         }
621     }
622
623     acquire_lock(&page_store_lock);
624
625     for (i = page_stores.next; i != &page_stores; i = i->next)
626     {
627         list_entry_t *j;
628         page_store_t *store = CONTAINER_OF(i, page_store_t, link);
629
630         for (j = store->entry_list.next; j != &store->entry_list; j = j->next)
631         {
632             page_store_entry_t *entry = CONTAINER_OF(j, page_store_entry_t, link);
633
634             if (entry->address_space == block->address_space
635                 && (dword_t)entry->address >= start_address
636                 && ((dword_t)entry->address < end_address))
637             {
638                 if (entry->number != INVALID_STORE_NUMBER) clear_bit(store->bitmap, entry->number);
639                 list_remove(&entry->link);
640                 free(entry);
641             }
642         }
643     }
644
645     release_lock(&page_store_lock);
646 }
647
648 static void free_blocks_recursive(memory_block_t *block)
649 {
650     release_memory_block(block);
651
652     if (block->by_addr_node.left)
653     {
654         memory_block_t *left_block = CONTAINER_OF(block->by_addr_node.left, memory_block_t, by_addr_node);
655         free_blocks_recursive(left_block);
656     }
657
658     if (block->by_addr_node.right)
659     {
660         memory_block_t *right_block = CONTAINER_OF(block->by_addr_node.right, memory_block_t, by_addr_node);
661         free_blocks_recursive(right_block);
662     }
663
664     mem_tree_free(block);
665 }
666
667 static memory_block_t *find_free_block_internal(memory_block_t *root, void *address, dword_t size)
668 {
669     avl_node_t *ptr;
670
671     if (root->by_size_node.left && (dword_t)root->size > size)
672     {
673         memory_block_t *left = CONTAINER_OF(root->by_size_node.left, memory_block_t, by_size_node);
674         memory_block_t *block = find_free_block_internal(left, address, size);
675         if (block) return block;
676     }
677
678     if ((dword_t)root->size >= size)
679     {
680         for (ptr = &root->by_size_node; ptr != NULL; ptr = ptr->next_equal)
681         {
682             memory_block_t *block = CONTAINER_OF(ptr, memory_block_t, by_size_node);
683
684             if (!(block->flags & MEMORY_BLOCK_FREE)) continue;
685
686             if (address != NULL)
687             {
688                 dword_t block_start = (dword_t)block->address;
689                 dword_t block_end = block_start + ((dword_t)block->size * PAGE_SIZE) - 1;
690
691                 dword_t needed_start = (dword_t)address;
692                 dword_t needed_end = needed_start + (size * PAGE_SIZE) - 1;
693
694                 if ((needed_start < block_start) || (needed_end > block_end)) continue;
695             }
696
697             return block;
698         }
699     }
700
701     if (!root->by_size_node.right) return NULL;
702     memory_block_t *right = CONTAINER_OF(root->by_size_node.right, memory_block_t, by_size_node);
703     return find_free_block_internal(right, address, size);
704 }
705
706 static memory_block_t *find_free_block(memory_address_space_t *address_space, void *address, dword_t size)
707 {
708     memory_block_t *root_block = CONTAINER_OF(address_space->by_size_tree.root, memory_block_t, by_size_node);
709     return find_free_block_internal(root_block, address, size);
710 }
711
712 static void *create_page_directory(void)
713 {
714     dword_t *current = (dword_t*)PAGE_DIRECTORY_ADDR;
715     dword_t new_dir_buffer[PAGE_SIZE / sizeof(dword_t)];
716
717     memset(&new_dir_buffer[USER_PAGE_START],
718            0,
719            (USER_PAGE_END - USER_PAGE_START + 1) * sizeof(dword_t));
720
721     memcpy(&new_dir_buffer[KERNEL_PAGE_START],
722            &current[KERNEL_PAGE_START],
723            (KERNEL_PAGE_END - KERNEL_PAGE_START + 1) * sizeof(dword_t));
724
725     void *directory = alloc_physical_page();
726     if (directory == NULL) return NULL;
727
728     new_dir_buffer[PAGEDIR_SELF_ENTRY] = (dword_t)directory | PAGE_PRESENT | PAGE_WRITABLE;
729     write_physical(directory, new_dir_buffer, PAGE_SIZE);
730
731     return directory;
732 }
733
734 static void fix_overlapping_sections(multiboot_mmap_t *mmap_addr, dword_t mmap_length)
735 {
736     multiboot_mmap_t *mmap = mmap_addr;
737
738     while ((dword_t)mmap < (dword_t)mmap_addr + mmap_length)
739     {
740         multiboot_mmap_t *ptr = (multiboot_mmap_t*)mmap_addr;
741
742         while ((dword_t)ptr < (dword_t) mmap)
743         {
744             qword_t mmap_end = mmap->base + mmap->length;
745             qword_t ptr_end = ptr->base + ptr->length;
746
747             if (mmap->base > ptr->base && mmap->base < ptr_end)
748             {
749                 mmap->base = ptr_end;
750                 if (mmap->base >= mmap_end) mmap->length = 0;
751                 else mmap->length = mmap_end - mmap->base;
752             }
753             else if (ptr->base > mmap->base && ptr->base < mmap_end)
754             {
755                 ptr->base = mmap_end;
756                 if (ptr->base >= ptr_end) ptr->length = 0;
757                 else ptr->length = ptr_end - ptr->base;
758             }
759
760             ptr = (multiboot_mmap_t*)((dword_t)ptr + ptr->size + sizeof(dword_t));
761         }
762
763         mmap = (multiboot_mmap_t*)((dword_t)mmap + mmap->size + sizeof(dword_t));
764     }
765 }
766
767 static inline memory_block_t *combine_blocks_forward(memory_block_t *mem_block)
768 {
769     while (TRUE)
770     {
771         avl_node_t *next = avl_get_next_node(&mem_block->by_addr_node);
772         if (!next) break;
773
774         memory_block_t *next_block = CONTAINER_OF(next, memory_block_t, by_addr_node);
775         if (!(next_block->flags & MEMORY_BLOCK_FREE)) break;
776
777         size_t new_size = mem_block->size + next_block->size;
778         avl_tree_change_key(&mem_block->address_space->by_size_tree, &mem_block->by_size_node, &new_size);
779
780         avl_tree_remove(&mem_block->address_space->by_addr_tree, &next_block->by_addr_node);
781         avl_tree_remove(&mem_block->address_space->by_size_tree, &next_block->by_size_node);
782         mem_tree_free(next_block);
783     }
784
785     return mem_block;
786 }
787
788 static inline memory_block_t *combine_blocks_backward(memory_block_t *mem_block)
789 {
790     while (TRUE)
791     {
792         avl_node_t *previous = avl_get_previous_node(&mem_block->by_addr_node);
793         if (!previous) break;
794
795         memory_block_t *prev_block = CONTAINER_OF(previous, memory_block_t, by_addr_node);
796         if (!(prev_block->flags & MEMORY_BLOCK_FREE)) break;
797
798         size_t new_size = prev_block->size + mem_block->size;
799         avl_tree_change_key(&mem_block->address_space->by_size_tree, &prev_block->by_size_node, &new_size);
800
801         avl_tree_remove(&mem_block->address_space->by_addr_tree, &mem_block->by_addr_node);
802         avl_tree_remove(&mem_block->address_space->by_size_tree, &mem_block->by_size_node);
803         mem_tree_free(mem_block);
804
805         mem_block = prev_block;
806     }
807
808     return mem_block;
809 }
810
811 void memory_cleanup(object_t *obj)
812 {
813     memory_section_t *section = (memory_section_t*)obj;
814     if (section->file) dereference(&section->file->header);
815 }
816
817 void *get_page_directory(void)
818 {
819     return current_page_directory;
820 }
821
822 void set_page_directory(void *phys_addr)
823 {
824     current_page_directory = phys_addr;
825
826     asm volatile ("mov %0, %%eax\n\
827                    mov %%eax, %%cr3" :: "r"(phys_addr));
828 }
829
830 void *get_physical_address(void *virtual)
831 {
832     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
833     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
834     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
835     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
836
837     if (!(page_directory[pd_index] & PAGE_PRESENT)) return INVALID_PAGE;
838     if (!(page_table[pt_index] & PAGE_PRESENT)) return INVALID_PAGE;
839
840     return (void*)PAGE_ALIGN(page_table[pt_index]);
841 }
842
843 dword_t map_memory_internal(void *physical, void *virtual, uintptr_t size, dword_t page_flags)
844 {
845     dword_t i, j;
846     dword_t phys_addr = PAGE_ALIGN((dword_t)physical);
847     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
848     size = PAGE_ALIGN_UP(size);
849     page_flags &= 0xFFF;
850
851     for (i = 0; i < size; i += PAGE_SIZE)
852     {
853         dword_t ret = map_page((void*)(phys_addr + i), (void*)(virt_addr + i), page_flags);
854         if (ret != ERR_SUCCESS)
855         {
856             for (j = 0; j < i; j += PAGE_SIZE) unmap_page((void*)(virt_addr + j));
857             release_resource(&mapping_space.resource);
858             return ret;
859         }
860     }
861
862     return ERR_SUCCESS;
863 }
864
865 void unmap_memory_internal(void *virtual, dword_t size)
866 {
867     dword_t i;
868     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
869     size = PAGE_ALIGN_UP(size);
870
871     for (i = 0; i < size; i += PAGE_SIZE)
872     {
873         void *page_addr = (void*)(virt_addr + i);
874         void *physical = get_physical_address(page_addr);
875
876         unmap_page(page_addr);
877         dereference_page(physical);
878     }
879 }
880
881 dword_t map_memory_in_address_space(memory_address_space_t *address_space,
882                                     void *physical,
883                                     void **virtual,
884                                     uintptr_t size,
885                                     dword_t block_flags)
886 {
887     dword_t ret;
888     void *address = (void*)PAGE_ALIGN((uintptr_t)*virtual);
889
890     acquire_resource_exclusive(&address_space->resource);
891
892     memory_block_t *block = find_free_block(address_space, address, size);
893     if (block == NULL)
894     {
895         release_resource(&address_space->resource);
896         return ERR_NOMEMORY;
897     }
898
899     dword_t flags = PAGE_GLOBAL;
900     dword_t real_address = (address != NULL) ? (dword_t)address : (dword_t)block->address;
901
902     block_flags &= ~MEMORY_BLOCK_EVICTABLE;
903     if (block_flags & MEMORY_BLOCK_ACCESSIBLE) flags |= PAGE_PRESENT;
904     if (block_flags & MEMORY_BLOCK_WRITABLE) flags |= PAGE_WRITABLE;
905     if (block_flags & MEMORY_BLOCK_USERMODE) flags |= PAGE_USERMODE;
906
907     ret = map_memory_internal(physical, (void*)real_address, size, flags);
908     if (ret != ERR_SUCCESS)
909     {
910         release_resource(&address_space->resource);
911         return ret;
912     }
913
914     if ((dword_t)block->address < (dword_t)address)
915     {
916         memory_block_t *new_block = mem_tree_alloc();
917         new_block->flags = MEMORY_BLOCK_FREE;
918         new_block->address = block->address;
919         new_block->size = (size_t)(((dword_t)address - block->address) / PAGE_SIZE);
920         new_block->address_space = address_space;
921         new_block->section = NULL;
922
923         size_t new_size = block->size - new_block->size;
924         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &new_size);
925         avl_tree_change_key(&address_space->by_addr_tree, &block->by_addr_node, &address);
926
927         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
928         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
929
930         combine_blocks_backward(new_block);
931     }
932
933     if ((dword_t)block->size > size)
934     {
935         memory_block_t *new_block = mem_tree_alloc();
936         new_block->flags = MEMORY_BLOCK_FREE;
937         new_block->address = (qword_t)(block->address + (size * PAGE_SIZE));
938         new_block->size = (qword_t)((dword_t)block->size - size);
939         new_block->address_space = address_space;
940         new_block->section = NULL;
941
942         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &size);
943
944         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
945         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
946
947         combine_blocks_forward(new_block);
948     }
949
950     block->flags = block_flags;
951     *virtual = (void*)((dword_t)block->address);
952
953     release_resource(&address_space->resource);
954     return ERR_SUCCESS;
955 }
956
957 dword_t pin_memory(const void *virtual, void **pinned, uintptr_t size, bool_t lock_contents)
958 {
959     uintptr_t i;
960     uintptr_t virt_addr = PAGE_ALIGN((uintptr_t)virtual);
961     void *address = (void*)PAGE_ALIGN((uintptr_t)*pinned);
962     size = PAGE_ALIGN_UP(size);
963
964     memory_address_space_t *address_space = check_usermode(virtual, 1) ? &get_current_process()->memory_space : &kernel_address_space;
965     acquire_resource_shared(&address_space->resource);
966     acquire_resource_exclusive(&mapping_space.resource);
967
968     memory_block_t *block = find_free_block(&mapping_space, address, size);
969     if (block == NULL)
970     {
971         release_resource(&address_space->resource);
972         release_resource(&mapping_space.resource);
973         return ERR_NOMEMORY;
974     }
975
976     dword_t real_address = (address != NULL) ? (dword_t)address : (dword_t)block->address;
977     dword_t new_flags = PAGE_PRESENT | PAGE_GLOBAL;
978     if (!lock_contents) new_flags |= PAGE_WRITABLE;
979
980     for (i = 0; i < size; i += PAGE_SIZE)
981     {
982         void *virt_page = (void*)(virt_addr + i);
983         void *phys_page = get_physical_address(virt_page);
984
985         if (lock_contents)
986         {
987             memory_block_t *block = find_block_by_addr(address_space, (void*)(virt_addr + i));
988             ASSERT(block != NULL);
989             block->flags |= MEMORY_BLOCK_COPY_ON_WRITE;
990             set_page_flags(virt_page, get_page_flags(virt_page) & ~PAGE_WRITABLE);
991         }
992
993         dword_t ret = map_page(phys_page, (void*)(real_address + i), new_flags);
994         ASSERT(ret == ERR_SUCCESS);
995         reference_page(phys_page);
996     }
997
998     if ((dword_t)block->address < (dword_t)address)
999     {
1000         memory_block_t *new_block = mem_tree_alloc();
1001         new_block->flags = MEMORY_BLOCK_FREE;
1002         new_block->address = block->address;
1003         new_block->size = (size_t)(((dword_t)address - block->address) / PAGE_SIZE);
1004         new_block->address_space = &mapping_space;
1005         new_block->section = NULL;
1006
1007         size_t new_size = block->size - new_block->size;
1008         avl_tree_change_key(&mapping_space.by_size_tree, &block->by_size_node, &new_size);
1009         avl_tree_change_key(&mapping_space.by_addr_tree, &block->by_addr_node, &address);
1010
1011         avl_tree_insert(&mapping_space.by_addr_tree, &new_block->by_addr_node);
1012         avl_tree_insert(&mapping_space.by_size_tree, &new_block->by_size_node);
1013
1014         combine_blocks_backward(new_block);
1015     }
1016
1017     if ((dword_t)block->size > size)
1018     {
1019         memory_block_t *new_block = mem_tree_alloc();
1020         new_block->flags = MEMORY_BLOCK_FREE;
1021         new_block->address = (qword_t)(block->address + (size * PAGE_SIZE));
1022         new_block->size = (qword_t)((dword_t)block->size - size);
1023         new_block->address_space = &mapping_space;
1024         new_block->section = NULL;
1025
1026         avl_tree_change_key(&mapping_space.by_size_tree, &block->by_size_node, &size);
1027
1028         avl_tree_insert(&mapping_space.by_addr_tree, &new_block->by_addr_node);
1029         avl_tree_insert(&mapping_space.by_size_tree, &new_block->by_size_node);
1030
1031         combine_blocks_forward(new_block);
1032     }
1033
1034     block->flags = MEMORY_BLOCK_ACCESSIBLE;
1035     if (!lock_contents) block->flags |= MEMORY_BLOCK_WRITABLE;
1036     *pinned = (void*)((dword_t)block->address) + PAGE_OFFSET((uintptr_t)virtual);
1037
1038     release_resource(&address_space->resource);
1039     release_resource(&mapping_space.resource);
1040     return ERR_SUCCESS;
1041 }
1042
1043 dword_t unmap_memory_in_address_space(memory_address_space_t *address_space, void *virtual)
1044 {
1045     acquire_resource_exclusive(&mapping_space.resource);
1046
1047     avl_node_t *node = avl_tree_lookup(&mapping_space.by_addr_tree, &virtual);
1048     if (node == NULL)
1049     {
1050         release_resource(&mapping_space.resource);
1051         return ERR_INVALID;
1052     }
1053
1054     memory_block_t *mem_block = CONTAINER_OF(node, memory_block_t, by_addr_node);
1055     if (mem_block->flags & MEMORY_BLOCK_FREE)
1056     {
1057         release_resource(&mapping_space.resource);
1058         return ERR_INVALID;
1059     }
1060
1061     unmap_memory_internal((void*)((dword_t)mem_block->address), (dword_t)mem_block->size);
1062
1063     mem_block->flags = MEMORY_BLOCK_FREE;
1064     mem_block = combine_blocks_backward(mem_block);
1065     mem_block = combine_blocks_forward(mem_block);
1066
1067     release_resource(&mapping_space.resource);
1068     return ERR_SUCCESS;
1069 }
1070
1071 dword_t map_memory(void *physical, void **virtual, uintptr_t size, dword_t block_flags)
1072 {
1073     return map_memory_in_address_space(&mapping_space, physical, virtual, size, block_flags);
1074 }
1075
1076 dword_t unmap_memory(void *virtual)
1077 {
1078     return unmap_memory_in_address_space(&mapping_space, virtual);
1079 }
1080
1081 dword_t alloc_memory_in_address_space(memory_address_space_t *address_space,
1082                                       void **address,
1083                                       dword_t size,
1084                                       dword_t block_flags,
1085                                       memory_section_t *section,
1086                                       qword_t section_offset)
1087 {
1088     void *base_address = (void*)PAGE_ALIGN((uintptr_t)*address);
1089
1090     block_flags &= ~(MEMORY_BLOCK_FREE | MEMORY_BLOCK_COPY_ON_WRITE);
1091     size = PAGE_ALIGN_UP(size) >> 12;
1092     if (size == 0) return ERR_INVALID;
1093
1094     acquire_resource_exclusive(&address_space->resource);
1095
1096     memory_block_t *block = find_free_block(address_space, base_address, size);
1097     if (block == NULL)
1098     {
1099         release_resource(&address_space->resource);
1100         return ERR_NOMEMORY;
1101     }
1102
1103     if (section)
1104     {
1105         reference(&section->header);
1106         block->section = section;
1107         block->section_offset = section_offset;
1108
1109         if ((section->flags & (MEMORY_SECTION_WRITABLE | MEMORY_SECTION_DIRECT_WRITE)) == MEMORY_SECTION_WRITABLE)
1110         {
1111             block_flags |= MEMORY_BLOCK_COPY_ON_WRITE;
1112         }
1113     }
1114
1115     if ((dword_t)block->address < (dword_t)base_address)
1116     {
1117         memory_block_t *new_block = mem_tree_alloc();
1118         new_block->flags = MEMORY_BLOCK_FREE;
1119         new_block->address = block->address;
1120         new_block->size = (size_t)(((dword_t)base_address - block->address) / PAGE_SIZE);
1121         new_block->address_space = address_space;
1122         new_block->section = NULL;
1123
1124         size_t new_size = block->size - new_block->size;
1125         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &new_size);
1126         avl_tree_change_key(&address_space->by_addr_tree, &block->by_addr_node, &base_address);
1127
1128         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
1129         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
1130
1131         combine_blocks_backward(new_block);
1132     }
1133
1134     if ((dword_t)block->size > size)
1135     {
1136         memory_block_t *new_block = mem_tree_alloc();
1137         new_block->flags = MEMORY_BLOCK_FREE;
1138         new_block->address = (qword_t)(block->address + (size * PAGE_SIZE));
1139         new_block->size = (qword_t)((dword_t)block->size - size);
1140         new_block->address_space = address_space;
1141         new_block->section = NULL;
1142
1143         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &size);
1144
1145         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
1146         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
1147
1148         combine_blocks_forward(new_block);
1149     }
1150
1151     block->flags = block_flags;
1152     *address = (void*)((dword_t)block->address);
1153     if (block_flags & MEMORY_BLOCK_EVICTABLE) list_append(&address_space->evictable_blocks, &block->evict_link);
1154
1155     release_resource(&address_space->resource);
1156     return ERR_SUCCESS;
1157 }
1158
1159 dword_t free_memory_in_address_space(memory_address_space_t *address_space, void *address)
1160 {
1161     acquire_resource_exclusive(&address_space->resource);
1162
1163     avl_node_t *node = avl_tree_lookup(&address_space->by_addr_tree, &address);
1164     if (node == NULL)
1165     {
1166         release_resource(&address_space->resource);
1167         return ERR_INVALID;
1168     }
1169
1170     memory_block_t *mem_block = CONTAINER_OF(node, memory_block_t, by_addr_node);
1171     if (mem_block->flags & MEMORY_BLOCK_FREE)
1172     {
1173         release_resource(&address_space->resource);
1174         return ERR_INVALID;
1175     }
1176
1177     release_memory_block(mem_block);
1178
1179     if (mem_block->flags & MEMORY_BLOCK_EVICTABLE) list_remove(&mem_block->evict_link);
1180     mem_block->flags = MEMORY_BLOCK_FREE;
1181
1182     mem_block = combine_blocks_backward(mem_block);
1183     mem_block = combine_blocks_forward(mem_block);
1184
1185     release_resource(&address_space->resource);
1186     return ERR_SUCCESS;
1187 }
1188
1189 dword_t commit_pages(void *address, size_t size)
1190 {
1191     uintptr_t i;
1192     uintptr_t first_page = PAGE_ALIGN((uintptr_t)address);
1193     uintptr_t last_page = PAGE_ALIGN_UP(first_page + size - 1);
1194
1195     EH_TRY
1196     {
1197         for (i = first_page; i <= last_page; i += PAGE_SIZE)
1198         {
1199             volatile uintptr_t value = *(volatile uintptr_t*)i;
1200             UNUSED_PARAMETER(value);
1201         }
1202     }
1203     EH_CATCH
1204     {
1205         EH_ESCAPE(return ERR_BADPTR);
1206     }
1207     EH_DONE;
1208
1209     return ERR_SUCCESS;
1210 }
1211
1212 dword_t uncommit_pages(void *address, size_t size)
1213 {
1214     uintptr_t i;
1215     uintptr_t first_page = PAGE_ALIGN((uintptr_t)address);
1216     uintptr_t last_page = PAGE_ALIGN_UP(first_page + size - 1);
1217
1218     EH_TRY
1219     {
1220         for (i = first_page; i <= last_page; i += PAGE_SIZE)
1221         {
1222             volatile uintptr_t value = *(volatile uintptr_t*)i;
1223             UNUSED_PARAMETER(value);
1224
1225             dword_t ret = unmap_page((void*)i);
1226             if (ret != ERR_SUCCESS) return ret;
1227         }
1228     }
1229     EH_CATCH
1230     {
1231         EH_ESCAPE(return ERR_BADPTR);
1232     }
1233     EH_DONE;
1234
1235     return ERR_SUCCESS;
1236 }
1237
1238 dword_t read_physical(void *physical, void *buffer, dword_t size)
1239 {
1240     critical_t critical;
1241     dword_t ret = ERR_SUCCESS;
1242     dword_t page;
1243     dword_t first_page = PAGE_ALIGN((dword_t)physical);
1244     dword_t last_page = PAGE_ALIGN((dword_t)physical + size - 1);
1245     dword_t offset = PAGE_OFFSET((dword_t)physical);
1246
1247     enter_critical(&critical);
1248
1249     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1250     {
1251         dword_t length = ((page == last_page) ? ((dword_t)physical + size - page) : PAGE_SIZE) - offset;
1252
1253         void *mapping = map_temporary_page((void*)page, PAGE_PRESENT);
1254         if (mapping == NULL) return ERR_NOMEMORY;
1255
1256         memcpy(buffer, (void*)((dword_t)mapping + offset), length);
1257         unmap_temporary_page(mapping);
1258
1259         buffer = (void*)((dword_t)buffer + length);
1260         offset = 0;
1261     }
1262
1263     leave_critical(&critical);
1264     return ret;
1265 }
1266
1267 dword_t write_physical(void *physical, void *buffer, dword_t size)
1268 {
1269     critical_t critical;
1270     dword_t ret = ERR_SUCCESS;
1271     dword_t page;
1272     dword_t first_page = PAGE_ALIGN((dword_t)physical);
1273     dword_t last_page = PAGE_ALIGN((dword_t)physical + size - 1);
1274     dword_t offset = PAGE_OFFSET((dword_t)physical);
1275
1276     enter_critical(&critical);
1277
1278     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1279     {
1280         dword_t length = ((page == last_page) ? ((dword_t)physical + size - page) : PAGE_SIZE) - offset;
1281
1282         void *mapping = map_temporary_page((void*)page, PAGE_PRESENT | PAGE_WRITABLE);
1283         if (mapping == NULL) return ERR_NOMEMORY;
1284
1285         memcpy((void*)((dword_t)mapping + offset), buffer, length);
1286         unmap_temporary_page(mapping);
1287
1288         buffer = (void*)((dword_t)buffer + length);
1289         offset = 0;
1290     }
1291
1292     leave_critical(&critical);
1293     return ret;
1294 }
1295
1296 sysret_t syscall_alloc_memory(handle_t process, void **address, dword_t size, dword_t flags)
1297 {
1298     process_t *proc;
1299     dword_t ret = ERR_SUCCESS;
1300     void *safe_address;
1301     void **local_address = address;
1302
1303     if (get_previous_mode() == USER_MODE)
1304     {
1305         flags &= MEMORY_BLOCK_WRITABLE | MEMORY_BLOCK_ACCESSIBLE;
1306         flags |= MEMORY_BLOCK_USERMODE | MEMORY_BLOCK_EVICTABLE;
1307
1308         if (!check_usermode(address, sizeof(void*))) return ERR_BADPTR;
1309
1310         EH_TRY
1311         {
1312             safe_address = *address;
1313             local_address = &safe_address;
1314         }
1315         EH_CATCH
1316         {
1317             EH_ESCAPE(return ERR_BADPTR);
1318         }
1319         EH_DONE;
1320     }
1321
1322     if (process != INVALID_HANDLE)
1323     {
1324         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1325     }
1326     else
1327     {
1328         proc = get_current_process();
1329         reference(&proc->header);
1330     }
1331
1332     ret = alloc_memory_in_address_space(&proc->memory_space, local_address, size, flags, NULL, 0ULL);
1333
1334     if (get_previous_mode() == USER_MODE)
1335     {
1336         EH_TRY *address = safe_address;
1337         EH_DONE;
1338     }
1339
1340     dereference(&proc->header);
1341     return ret;
1342 }
1343
1344 sysret_t syscall_free_memory(handle_t process, void *address)
1345 {
1346     dword_t ret = ERR_SUCCESS;
1347     process_t *proc;
1348
1349     if (process != INVALID_HANDLE)
1350     {
1351         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1352     }
1353     else
1354     {
1355         proc = get_current_process();
1356         reference(&proc->header);
1357     }
1358
1359     ret = free_memory_in_address_space(&proc->memory_space, address);
1360
1361     dereference(&proc->header);
1362     return ret;
1363 }
1364
1365 sysret_t syscall_commit_memory(handle_t process, void *address, dword_t size)
1366 {
1367     dword_t ret = ERR_SUCCESS;
1368     process_t *proc;
1369
1370     if (get_previous_mode() == USER_MODE && !check_usermode(address, size)) return ERR_BADPTR;
1371
1372     if (process == INVALID_HANDLE)
1373     {
1374         proc = get_current_process();
1375         reference(&proc->header);
1376     }
1377     else
1378     {
1379         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1380     }
1381
1382     if (proc->terminating) return ERR_CANCELED;
1383     acquire_resource_shared(&proc->memory_space.resource);
1384
1385     process_t *prev_proc = switch_process(proc);
1386     ret = commit_pages(address, size);
1387     switch_process(prev_proc);
1388
1389     release_resource(&proc->memory_space.resource);
1390     dereference(&proc->header);
1391     return ret;
1392 }
1393
1394 sysret_t syscall_uncommit_memory(handle_t process, void *address, dword_t size)
1395 {
1396     dword_t ret = ERR_SUCCESS;
1397     process_t *proc;
1398
1399     if (get_previous_mode() == USER_MODE && !check_usermode(address, size)) return ERR_BADPTR;
1400
1401     if (process == INVALID_HANDLE)
1402     {
1403         proc = get_current_process();
1404         reference(&proc->header);
1405     }
1406     else
1407     {
1408         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1409     }
1410
1411     if (proc->terminating) return ERR_CANCELED;
1412     acquire_resource_shared(&proc->memory_space.resource);
1413
1414     process_t *prev_proc = switch_process(proc);
1415     ret = uncommit_pages(address, size);
1416     switch_process(prev_proc);
1417
1418     release_resource(&proc->memory_space.resource);
1419     dereference(&proc->header);
1420     return ret;
1421 }
1422
1423 sysret_t syscall_set_memory_flags(handle_t process, void *address, dword_t flags)
1424 {
1425     dword_t ret = ERR_SUCCESS;
1426     process_t *proc;
1427
1428     flags &= ~(MEMORY_BLOCK_FREE | MEMORY_BLOCK_COPY_ON_WRITE);
1429     if (get_previous_mode() == USER_MODE) flags |= MEMORY_BLOCK_USERMODE | MEMORY_BLOCK_EVICTABLE;
1430
1431     if (process != INVALID_HANDLE)
1432     {
1433         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1434     }
1435     else
1436     {
1437         proc = get_current_process();
1438         reference(&proc->header);
1439     }
1440
1441     process_t *prev_proc = switch_process(proc);
1442     acquire_resource_exclusive(&proc->memory_space.resource);
1443
1444     memory_block_t *block = find_block_by_addr(&proc->memory_space, address);
1445     if (block == NULL)
1446     {
1447         ret = ERR_INVALID;
1448         goto cleanup;
1449     }
1450
1451     if (block->section)
1452     {
1453         if ((flags & MEMORY_BLOCK_WRITABLE) && !(block->section->flags & MEMORY_SECTION_WRITABLE))
1454         {
1455             ret = ERR_FORBIDDEN;
1456             goto cleanup;
1457         }
1458     }
1459
1460     if (block->flags & MEMORY_BLOCK_FREE)
1461     {
1462         ret = ERR_INVALID;
1463         goto cleanup;
1464     }
1465
1466     dword_t page;
1467     dword_t start_address = (dword_t)block->address;
1468     dword_t end_address = start_address + (dword_t)block->size * PAGE_SIZE;
1469     dword_t page_flags = 0;
1470
1471     if (flags & MEMORY_BLOCK_ACCESSIBLE) page_flags |= PAGE_PRESENT;
1472     if (flags & MEMORY_BLOCK_WRITABLE) page_flags |= PAGE_WRITABLE;
1473
1474     if (flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
1475     else page_flags |= PAGE_GLOBAL;
1476
1477     for (page = start_address; page < end_address; page += PAGE_SIZE)
1478     {
1479         set_page_flags((void*)page, page_flags);
1480     }
1481
1482     if (!(block->flags & MEMORY_BLOCK_EVICTABLE) && (flags & MEMORY_BLOCK_EVICTABLE))
1483     {
1484         list_append(&proc->memory_space.evictable_blocks, &block->evict_link);
1485     }
1486     else if ((block->flags & MEMORY_BLOCK_EVICTABLE) && !(flags & MEMORY_BLOCK_EVICTABLE))
1487     {
1488         list_remove(&block->evict_link);
1489     }
1490
1491     block->flags &= MEMORY_BLOCK_COPY_ON_WRITE;
1492     block->flags |= flags;
1493
1494 cleanup:
1495     release_resource(&proc->memory_space.resource);
1496     switch_process(prev_proc);
1497     dereference(&proc->header);
1498     return ret;
1499 }
1500
1501 sysret_t syscall_query_memory(handle_t process, void *address, memory_block_info_t *info)
1502 {
1503     dword_t ret = ERR_SUCCESS;
1504     process_t *proc;
1505
1506     if ((get_previous_mode() == USER_MODE) && !check_usermode(info, sizeof(memory_block_info_t)))
1507     {
1508         return ERR_BADPTR;
1509     }
1510
1511     if (process != INVALID_HANDLE)
1512     {
1513         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1514     }
1515     else
1516     {
1517         proc = get_current_process();
1518         reference(&proc->header);
1519     }
1520
1521     acquire_resource_shared(&proc->memory_space.resource);
1522
1523     memory_block_t *block = find_block_by_addr(&proc->memory_space, address);
1524     if (block == NULL)
1525     {
1526         ret = ERR_INVALID;
1527         goto cleanup;
1528     }
1529
1530     EH_TRY
1531     {
1532         info->address = block->address;
1533         info->size = block->size;
1534         info->flags = block->flags;
1535     }
1536     EH_CATCH
1537     {
1538         ret = ERR_BADPTR;
1539     }
1540     EH_DONE;
1541
1542 cleanup:
1543     release_resource(&proc->memory_space.resource);
1544     dereference(&proc->header);
1545     return ret;
1546 }
1547
1548 sysret_t syscall_read_memory(handle_t process, void *address, void *buffer, dword_t size)
1549 {
1550     dword_t ret = ERR_SUCCESS;
1551     process_t *proc;
1552     byte_t page_cache[PAGE_SIZE];
1553
1554     if (get_previous_mode() == USER_MODE && !check_usermode(buffer, size)) return ERR_BADPTR;
1555
1556     if (process == INVALID_HANDLE)
1557     {
1558         EH_TRY
1559         {
1560             memmove(buffer, address, size);
1561             return ERR_SUCCESS;
1562         }
1563         EH_CATCH
1564         {
1565             EH_ESCAPE(return ERR_FORBIDDEN);
1566         }
1567         EH_DONE;
1568     }
1569
1570     if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1571     if (proc->terminating) return ERR_CANCELED;
1572
1573     acquire_resource_shared(&proc->memory_space.resource);
1574
1575     dword_t page;
1576     dword_t first_page = PAGE_ALIGN((dword_t)address);
1577     dword_t last_page = PAGE_ALIGN((dword_t)address + size - 1);
1578     dword_t offset = PAGE_OFFSET((dword_t)address);
1579
1580     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1581     {
1582         dword_t length = ((page == last_page) ? ((dword_t)address + size - page) : PAGE_SIZE) - offset;
1583
1584         process_t *prev_proc = switch_process(proc);
1585
1586         EH_TRY memcpy(&page_cache[offset], (void*)(page + offset), length);
1587         EH_CATCH ret = ERR_FORBIDDEN;
1588         EH_DONE;
1589
1590         switch_process(prev_proc);
1591         if (ret != ERR_SUCCESS) break;
1592
1593         EH_TRY memcpy(buffer, &page_cache[offset], length);
1594         EH_CATCH ret = ERR_BADPTR;
1595         EH_DONE;
1596
1597         buffer = (void*)((dword_t)buffer + length);
1598         offset = 0;
1599         if (ret != ERR_SUCCESS) break;
1600     }
1601
1602     release_resource(&proc->memory_space.resource);
1603     dereference(&proc->header);
1604     return ret;
1605 }
1606
1607 sysret_t syscall_write_memory(handle_t process, void *address, void *buffer, dword_t size)
1608 {
1609     dword_t ret = ERR_SUCCESS;
1610     process_t *proc;
1611     byte_t page_cache[PAGE_SIZE];
1612
1613     if (get_previous_mode() == USER_MODE && !check_usermode(buffer, size)) return ERR_BADPTR;
1614
1615     if (process == INVALID_HANDLE)
1616     {
1617         EH_TRY
1618         {
1619             memmove(address, buffer, size);
1620             return ERR_SUCCESS;
1621         }
1622         EH_CATCH
1623         {
1624             EH_ESCAPE(return ERR_FORBIDDEN);
1625         }
1626         EH_DONE;
1627     }
1628
1629     if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1630     if (proc->terminating) return ERR_CANCELED;
1631
1632     acquire_resource_exclusive(&proc->memory_space.resource);
1633
1634     dword_t page;
1635     dword_t first_page = PAGE_ALIGN((dword_t)address);
1636     dword_t last_page = PAGE_ALIGN((dword_t)address + size - 1);
1637     dword_t offset = PAGE_OFFSET((dword_t)address);
1638
1639     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1640     {
1641         dword_t length = ((page == last_page) ? ((dword_t)address + size - page) : PAGE_SIZE) - offset;
1642
1643         EH_TRY memcpy(&page_cache[offset], buffer, length);
1644         EH_CATCH ret = ERR_BADPTR;
1645         EH_DONE;
1646
1647         if (ret != ERR_SUCCESS) break;
1648         process_t *prev_proc = switch_process(proc);
1649
1650         EH_TRY memcpy((void*)(page + offset), &page_cache[offset], length);
1651         EH_CATCH ret = ERR_FORBIDDEN;
1652         EH_DONE;
1653
1654         switch_process(prev_proc);
1655
1656         buffer = (void*)((dword_t)buffer + length);
1657         offset = 0;
1658         if (ret != ERR_SUCCESS) break;
1659     }
1660
1661     release_resource(&proc->memory_space.resource);
1662     dereference(&proc->header);
1663     return ret;
1664 }
1665
1666 void *alloc_pool(void *address, dword_t size, dword_t block_flags)
1667 {
1668     size = PAGE_ALIGN_UP(size);
1669     void *result = address;
1670
1671     if (alloc_memory_in_address_space(&kernel_address_space,
1672                                       &result,
1673                                       size,
1674                                       block_flags,
1675                                       NULL,
1676                                       0ULL) == ERR_SUCCESS)
1677     {
1678         return result;
1679     }
1680     else
1681     {
1682         return NULL;
1683     }
1684 }
1685
1686 void free_pool(void *address)
1687 {
1688     free_memory_in_address_space(&kernel_address_space, address);
1689 }
1690
1691 sysret_t syscall_create_memory_section(const char *name, handle_t file, size_t max_size, dword_t flags, handle_t *handle)
1692 {
1693     dword_t ret = ERR_SUCCESS;
1694     handle_t safe_handle;
1695     char *safe_name = NULL;
1696
1697     flags &= MEMORY_SECTION_WRITABLE | MEMORY_SECTION_DIRECT_WRITE;
1698     if (flags & MEMORY_SECTION_DIRECT_WRITE) flags |= MEMORY_SECTION_WRITABLE;
1699
1700     if (get_previous_mode() == USER_MODE)
1701     {
1702         dword_t name_length = 0;
1703
1704         EH_TRY name_length = strlen(name);
1705         EH_CATCH EH_ESCAPE(return ERR_BADPTR);
1706         EH_DONE;
1707
1708         if (!check_usermode(name, name_length + 1)) return ERR_BADPTR;
1709         if (!check_usermode(handle, sizeof(handle_t))) return ERR_BADPTR;
1710
1711         safe_name = copy_user_string(name);
1712         if (safe_name == NULL) return ERR_BADPTR;
1713     }
1714     else
1715     {
1716         safe_name = (char*)name;
1717     }
1718
1719     memory_section_t *section = (memory_section_t*)malloc(sizeof(memory_section_t));
1720     if (section == NULL)
1721     {
1722         ret = ERR_NOMEMORY;
1723         goto cleanup;
1724     }
1725
1726     file_instance_t *file_instance = NULL;
1727     if (file != INVALID_HANDLE)
1728     {
1729         if (!reference_by_handle(file, OBJECT_FILE_INSTANCE, (object_t**)&file_instance))
1730         {
1731             ret = ERR_INVALID;
1732             goto cleanup;
1733         }
1734     }
1735
1736     list_init(&section->page_list);
1737     section->flags = flags;
1738     section->size = max_size;
1739     section->file = file != INVALID_HANDLE ? file_instance : NULL;
1740
1741     init_object(&section->header, safe_name, OBJECT_MEMORY);
1742     ret = create_object(&section->header);
1743     if (ret != ERR_SUCCESS)
1744     {
1745         if (file_instance) dereference(&file_instance->header);
1746         if (section->header.name) free(section->header.name);
1747         free(section);
1748         section = NULL;
1749         goto cleanup;
1750     }
1751
1752     ret = open_object(&section->header, 0, &safe_handle);
1753     if (ret == ERR_SUCCESS)
1754     {
1755         EH_TRY
1756         {
1757             *handle = safe_handle;
1758         }
1759         EH_CATCH
1760         {
1761             syscall_close_object(safe_handle);
1762             ret = ERR_BADPTR;
1763         }
1764         EH_DONE;
1765     }
1766
1767 cleanup:
1768     if (section) dereference(&section->header);
1769     if (get_previous_mode() == USER_MODE) free(safe_name);
1770
1771     return ret;
1772 }
1773
1774 sysret_t syscall_open_memory_section(const char *name, handle_t *handle)
1775 {
1776     handle_t safe_handle;
1777     char *safe_name = NULL;
1778
1779     if (get_previous_mode() == USER_MODE)
1780     {
1781         dword_t name_length = 0;
1782
1783         EH_TRY name_length = strlen(name);
1784         EH_CATCH EH_ESCAPE(return ERR_BADPTR);
1785         EH_DONE;
1786
1787         if (!check_usermode(name, name_length + 1)) return ERR_BADPTR;
1788         if (!check_usermode(handle, sizeof(handle_t))) return ERR_BADPTR;
1789
1790         safe_name = copy_user_string(name);
1791         if (safe_name == NULL) return ERR_NOMEMORY;
1792     }
1793     else safe_name = (char*)name;
1794
1795     dword_t ret = open_object_by_name(safe_name, OBJECT_MEMORY, 0, &safe_handle);
1796
1797     EH_TRY
1798     {
1799         *handle = safe_handle;
1800     }
1801     EH_CATCH
1802     {
1803         syscall_close_object(safe_handle);
1804         ret = ERR_BADPTR;
1805     }
1806     EH_DONE;
1807
1808     if (get_previous_mode() == USER_MODE) free(safe_name);
1809     return ret;
1810 }
1811
1812 sysret_t syscall_map_memory_section(handle_t process, handle_t section, void **address, qword_t offset, size_t size, dword_t flags)
1813 {
1814     dword_t ret = ERR_SUCCESS;
1815     process_t *proc = NULL;
1816     memory_section_t *mem_sec = NULL;
1817     void *safe_address;
1818
1819     if (PAGE_OFFSET(offset) != 0) return ERR_INVALID;
1820
1821     if (process != INVALID_HANDLE)
1822     {
1823         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc))
1824         {
1825             ret = ERR_INVALID;
1826             goto cleanup;
1827         }
1828     }
1829     else
1830     {
1831         proc = get_current_process();
1832         reference(&proc->header);
1833     }
1834
1835     if (!reference_by_handle(section, OBJECT_MEMORY, (object_t**)&mem_sec))
1836     {
1837         ret = ERR_INVALID;
1838         goto cleanup;
1839     }
1840
1841     if (get_previous_mode() == USER_MODE)
1842     {
1843         if (!check_usermode(address, sizeof(void*)))
1844         {
1845             ret = ERR_BADPTR;
1846             goto cleanup;
1847         }
1848
1849         EH_TRY safe_address = *address;
1850         EH_CATCH ret = ERR_BADPTR;
1851         EH_DONE;
1852
1853         if (ret != ERR_SUCCESS) goto cleanup;
1854     }
1855     else
1856     {
1857         safe_address = *address;
1858     }
1859
1860     if ((flags & MEMORY_BLOCK_WRITABLE) && !(mem_sec->flags & MEMORY_SECTION_WRITABLE))
1861     {
1862         ret = ERR_FORBIDDEN;
1863         goto cleanup;
1864     }
1865
1866     ret = alloc_memory_in_address_space(&proc->memory_space, &safe_address, size, flags, mem_sec, offset);
1867     if (ret != ERR_SUCCESS) goto cleanup;
1868
1869     EH_TRY *address = safe_address;
1870     EH_DONE;
1871
1872 cleanup:
1873     if (proc) dereference(&proc->header);
1874     if (mem_sec) dereference(&mem_sec->header);
1875     return ret;
1876 }
1877
1878 sysret_t syscall_flush_memory_section(handle_t process, void *address)
1879 {
1880     dword_t ret = ERR_SUCCESS;
1881     process_t *proc = NULL;
1882
1883     if (process != INVALID_HANDLE)
1884     {
1885         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc))
1886         {
1887             ret = ERR_INVALID;
1888             goto cleanup;
1889         }
1890     }
1891     else
1892     {
1893         proc = get_current_process();
1894         reference(&proc->header);
1895     }
1896
1897     acquire_resource_shared(&proc->memory_space.resource);
1898
1899     memory_block_t *block = find_block_by_addr(&proc->memory_space, address);
1900     if (block == NULL || block->section == NULL)
1901     {
1902         ret = ERR_INVALID;
1903         goto cleanup;
1904     }
1905
1906     if (block->section->file == NULL) goto cleanup;
1907
1908     list_entry_t *ptr;
1909
1910     for (ptr = block->section->page_list.next; ptr != &block->section->page_list; ptr = ptr->next)
1911     {
1912         dword_t bytes_written;
1913         byte_t buffer[PAGE_SIZE];
1914         shared_page_t *shared = CONTAINER_OF(ptr, shared_page_t, link);
1915
1916         ret = read_physical(shared->physical, buffer, PAGE_SIZE);
1917         if (ret != ERR_SUCCESS) continue;
1918
1919         file_instance_t *file = block->section->file;
1920         acquire_resource_exclusive(&file->global->volume->resource);
1921         ret = file->global->volume->driver->write_file(file, buffer, shared->offset, PAGE_SIZE, &bytes_written);
1922         release_resource(&file->global->volume->resource);
1923         if (ret != ERR_SUCCESS) break;
1924     }
1925
1926 cleanup:
1927     release_resource(&proc->memory_space.resource);
1928     dereference(&proc->header);
1929     return ret;
1930 }
1931
1932 sysret_t syscall_add_page_file(const char *path, dword_t max_entries)
1933 {
1934     dword_t ret;
1935     char *safe_path = NULL;
1936     if (max_entries == INVALID_STORE_NUMBER) max_entries--;
1937
1938     if (get_previous_mode() == USER_MODE)
1939     {
1940         if (!check_privileges(PRIVILEGE_SET_PAGE_FILE)) return ERR_FORBIDDEN;
1941
1942         if (path)
1943         {
1944             dword_t path_length = 0;
1945
1946             EH_TRY path_length = strlen(path);
1947             EH_CATCH EH_ESCAPE(return ERR_BADPTR);
1948             EH_DONE;
1949
1950             if (!check_usermode(path, path_length + 1)) return ERR_BADPTR;
1951
1952             safe_path = copy_user_string(path);
1953             if (!safe_path) return ERR_NOMEMORY;
1954         }
1955     }
1956     else safe_path = (char*)path;
1957
1958     page_store_t *store = (page_store_t*)malloc(sizeof(page_store_t));
1959     if (store == NULL)
1960     {
1961         ret = ERR_NOMEMORY;
1962         goto cleanup;
1963     }
1964
1965     store->bitmap = malloc((max_entries + 7) / 8);
1966     if (store->bitmap == NULL)
1967     {
1968         free(store);
1969         ret = ERR_NOMEMORY;
1970         goto cleanup;
1971     }
1972
1973     memset(store->bitmap, 0, (max_entries + 7) / 8);
1974     store->num_entries = 0;
1975     store->max_entries = max_entries;
1976     list_init(&store->entry_list);
1977
1978     ret = syscall(SYSCALL_OPEN_FILE,
1979                   safe_path,
1980                   &store->file_handle,
1981                   FILE_MODE_READ
1982                   | FILE_MODE_WRITE
1983                   | FILE_MODE_NO_CACHE
1984                   | FILE_MODE_DELETE_ON_CLOSE
1985                   | FILE_MODE_CREATE
1986                   | FILE_MODE_TRUNCATE,
1987                   0);
1988     if (ret != ERR_SUCCESS)
1989     {
1990         free(store->bitmap);
1991         free(store);
1992         goto cleanup;
1993     }
1994
1995     acquire_lock(&page_store_lock);
1996     list_append(&page_stores, &store->link);
1997     release_lock(&page_store_lock);
1998
1999 cleanup:
2000     if (get_previous_mode() == USER_MODE) free(safe_path);
2001     return ret;
2002 }
2003
2004 sysret_t syscall_remove_page_file(const char *path)
2005 {
2006     dword_t ret = ERR_SUCCESS;
2007     char *safe_path = NULL;
2008
2009     if (get_previous_mode() == USER_MODE)
2010     {
2011         if (!check_privileges(PRIVILEGE_SET_PAGE_FILE)) return ERR_FORBIDDEN;
2012
2013         if (path)
2014         {
2015             dword_t path_length = 0;
2016
2017             EH_TRY path_length = strlen(path);
2018             EH_CATCH EH_ESCAPE(return ERR_BADPTR);
2019             EH_DONE;
2020
2021             if (!check_usermode(path, path_length + 1)) return ERR_BADPTR;
2022
2023             safe_path = copy_user_string(path);
2024             if (!safe_path) return ERR_NOMEMORY;
2025         }
2026     }
2027     else safe_path = (char*)path;
2028
2029     list_entry_t *ptr;
2030     page_store_t *store;
2031
2032     acquire_lock(&page_store_lock);
2033
2034     for (ptr = page_stores.next; ptr != &page_stores; ptr = ptr->next)
2035     {
2036         store = CONTAINER_OF(ptr, page_store_t, link);
2037
2038         char *name_buffer = NULL;
2039         size_t name_buffer_size = 256;
2040
2041         while (TRUE)
2042         {
2043             char *name_buffer = malloc(name_buffer_size);
2044             if (!name_buffer) break;
2045
2046             ret = syscall(SYSCALL_QUERY_FILE, store->file_handle, name_buffer, name_buffer_size);
2047             if (ret != ERR_SUCCESS) free(name_buffer);
2048             if (ret != ERR_SMALLBUF) break;
2049
2050             name_buffer_size *= 2;
2051         }
2052
2053         if (ret == ERR_SUCCESS)
2054         {
2055             bool_t found = strcmp(name_buffer, safe_path) == 0;
2056             if (name_buffer) free(name_buffer);
2057             if (found) break;
2058         }
2059     }
2060
2061     if (ptr == &page_stores)
2062     {
2063         ret = ERR_NOTFOUND;
2064         release_lock(&page_store_lock);
2065         goto cleanup;
2066     }
2067
2068     list_remove(&store->link);
2069     release_lock(&page_store_lock);
2070
2071     for (ptr = store->entry_list.next; ptr != &store->entry_list; ptr = ptr->next)
2072     {
2073         process_t *old_process;
2074         byte_t buffer[PAGE_SIZE];
2075         dword_t bytes_read;
2076         dword_t page_flags = 0;
2077         page_store_entry_t *entry = CONTAINER_OF(ptr, page_store_entry_t, link);
2078
2079         ret = syscall_read_file(store->file_handle, buffer, (qword_t)entry->number * (qword_t)PAGE_SIZE, PAGE_SIZE, &bytes_read);
2080         if (ret != ERR_SUCCESS) break;
2081
2082         acquire_resource_exclusive(&entry->address_space->resource);
2083         memory_block_t *block = find_block_by_addr(entry->address_space, entry->address);
2084
2085         if (block->flags & MEMORY_BLOCK_ACCESSIBLE) page_flags |= PAGE_PRESENT;
2086         if ((block->flags & (MEMORY_BLOCK_WRITABLE | MEMORY_BLOCK_COPY_ON_WRITE))
2087             == MEMORY_BLOCK_WRITABLE)
2088         {
2089             page_flags |= PAGE_WRITABLE;
2090         }
2091
2092         if (block->flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
2093         else page_flags |= PAGE_GLOBAL;
2094
2095         if (entry->address_space != &kernel_address_space)
2096         {
2097             old_process = switch_process(CONTAINER_OF(entry->address_space, process_t, memory_space));
2098         }
2099
2100         ret = alloc_page(entry->address, page_flags);
2101         if (ret != ERR_SUCCESS) goto loop_cleanup;
2102
2103         list_entry_t *p;
2104         for (p = store->entry_list.next; p != &store->entry_list; p = ptr->next)
2105         {
2106             page_store_entry_t *other_entry = CONTAINER_OF(ptr, page_store_entry_t, link);
2107
2108             if (entry != other_entry && other_entry->number == entry->number)
2109             {
2110                 list_remove(&other_entry->link);
2111                 list_append(&transition_pages, &other_entry->link);
2112
2113                 other_entry->physical = get_physical_address(entry->address);
2114                 other_entry->number = INVALID_STORE_NUMBER;
2115             }
2116         }
2117
2118         clear_bit(store->bitmap, entry->number);
2119         list_remove(&entry->link);
2120
2121         memcpy(entry->address, buffer, PAGE_SIZE);
2122         free(entry);
2123
2124 loop_cleanup:
2125         if (entry->address_space != &kernel_address_space) switch_process(old_process);
2126         release_resource(&entry->address_space->resource);
2127     }
2128
2129     free(store);
2130
2131 cleanup:
2132     if (ret != ERR_SUCCESS)
2133     {
2134         acquire_lock(&page_store_lock);
2135         list_append(&page_stores, &store->link);
2136         release_lock(&page_store_lock);
2137     }
2138
2139     if (get_previous_mode() == USER_MODE) free(safe_path);
2140     return ret;
2141 }
2142
2143 static int compare_address(const void *key1, const void *key2)
2144 {
2145     const uintptr_t first = *(const uintptr_t*)key1;
2146     const uintptr_t second = *(const uintptr_t*)key2;
2147
2148     if (first < second) return -1;
2149     else if (first == second) return 0;
2150     else return 1;
2151 }
2152
2153 static int compare_size(const void *key1, const void *key2)
2154 {
2155     const size_t first = *(const size_t*)key1;
2156     const size_t second = *(const size_t*)key2;
2157
2158     if (first < second) return -1;
2159     else if (first == second) return 0;
2160     else return 1;
2161 }
2162
2163 dword_t create_address_space(void *base_address, dword_t page_count, memory_address_space_t *mem_space)
2164 {
2165     dword_t ret = ERR_NOMEMORY;
2166
2167     mem_space->pool_address = base_address;
2168     mem_space->pool_size = page_count;
2169     AVL_TREE_INIT(&mem_space->by_addr_tree, memory_block_t, by_addr_node, address, compare_address);
2170     AVL_TREE_INIT(&mem_space->by_size_tree, memory_block_t, by_size_node, size, compare_size);
2171     mem_space->resource = 0;
2172     list_init(&mem_space->evictable_blocks);
2173     mem_space->evict_blk_ptr = NULL;
2174     mem_space->evict_page_num = 0;
2175     mem_space->stats.used_virtual = 0;
2176     mem_space->stats.committed = 0;
2177     mem_space->stats.evicted = 0;
2178     mem_space->stats.shared = 0;
2179
2180     if (get_page_directory() != INVALID_PAGE)
2181     {
2182         mem_space->page_directory = create_page_directory();
2183         if (mem_space->page_directory == NULL) return ret;
2184     }
2185     else
2186     {
2187         dword_t *boot_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
2188         mem_space->page_directory = (void*)PAGE_ALIGN(boot_directory[PAGEDIR_SELF_ENTRY]);
2189     }
2190
2191     memory_block_t *initial = mem_tree_alloc();
2192     if (initial != NULL)
2193     {
2194         initial->address = (uintptr_t)base_address;
2195         initial->size = page_count;
2196         initial->flags = MEMORY_BLOCK_FREE;
2197         initial->address_space = mem_space;
2198         initial->section = NULL;
2199
2200         avl_tree_insert(&mem_space->by_addr_tree, &initial->by_addr_node);
2201         avl_tree_insert(&mem_space->by_size_tree, &initial->by_size_node);
2202         ret = ERR_SUCCESS;
2203     }
2204
2205     if (mem_space != &kernel_address_space)
2206     {
2207         list_append(&user_address_spaces, &mem_space->link);
2208     }
2209
2210     return ret;
2211 }
2212
2213 dword_t clone_address_space(memory_address_space_t *original, memory_address_space_t *clone)
2214 {
2215     dword_t i;
2216     dword_t ret = ERR_SUCCESS;
2217
2218     acquire_resource_shared(&original->resource);
2219
2220     clone->pool_address = original->pool_address;
2221     clone->pool_size = original->pool_size;
2222     AVL_TREE_INIT(&clone->by_addr_tree, memory_block_t, by_addr_node, address, NULL);
2223     AVL_TREE_INIT(&clone->by_size_tree, memory_block_t, by_size_node, size, NULL);
2224     clone->resource = 0;
2225     list_init(&clone->evictable_blocks);
2226     clone->evict_blk_ptr = NULL;
2227     clone->evict_page_num = 0;
2228     clone->stats.used_virtual = original->stats.used_virtual;
2229     clone->stats.committed = original->stats.committed;
2230     clone->stats.evicted = original->stats.evicted;
2231     clone->stats.shared = original->stats.committed;
2232
2233     if (original->by_addr_tree.root != NULL)
2234     {
2235         memory_block_t *root_block = CONTAINER_OF(original->by_addr_tree.root, memory_block_t, by_addr_node);
2236         if (!clone_blocks_recursive(clone, root_block))
2237         {
2238             ret = ERR_NOMEMORY;
2239             goto cleanup;
2240         }
2241     }
2242
2243     if (!(clone->page_directory = create_page_directory()))
2244     {
2245         ret = ERR_NOMEMORY;
2246         goto cleanup;
2247     }
2248
2249     dword_t *clone_dir = map_temporary_page(clone->page_directory, PAGE_PRESENT | PAGE_WRITABLE);
2250     bool_t this_directory = original->page_directory == get_page_directory();
2251
2252     dword_t *original_dir;
2253     if (this_directory) original_dir = (dword_t*)PAGE_DIRECTORY_ADDR;
2254     else original_dir = map_temporary_page(original->page_directory, PAGE_PRESENT | PAGE_WRITABLE);
2255
2256     for (i = USER_PAGE_START; i <= USER_PAGE_END; i++)
2257     {
2258         reference_page((void*)PAGE_ALIGN(original_dir[i]));
2259         original_dir[i] &= ~PAGE_WRITABLE;
2260         clone_dir[i] = original_dir[i];
2261         if (this_directory) invalidate_tlb((void*)(i << 12));
2262     }
2263
2264     if (!this_directory) unmap_temporary_page(original_dir);
2265     unmap_temporary_page(clone_dir);
2266     list_append(&user_address_spaces, &clone->link);
2267
2268 cleanup:
2269     release_resource(&original->resource);
2270     return ret;
2271 }
2272
2273 void bump_address_space(memory_address_space_t *mem_space)
2274 {
2275     list_remove(&mem_space->link);
2276     list_append(&user_address_spaces, &mem_space->link);
2277 }
2278
2279 void delete_address_space(memory_address_space_t *mem_space)
2280 {
2281     ASSERT(get_page_directory() != mem_space->page_directory);
2282     acquire_resource_exclusive(&mem_space->resource);
2283
2284     if (mem_space->by_addr_tree.root)
2285     {
2286         memory_block_t *root = CONTAINER_OF(mem_space->by_addr_tree.root, memory_block_t, by_addr_node);
2287         free_blocks_recursive(root);
2288         mem_space->by_addr_tree.root = mem_space->by_size_tree.root = NULL;
2289     }
2290
2291     free_physical_page(mem_space->page_directory);
2292     mem_space->page_directory = NULL;
2293
2294     release_resource(&mem_space->resource);
2295 }
2296
2297 static bool_t find_evicted_page(memory_block_t *block, void *address, page_store_t **store, page_store_entry_t **entry)
2298 {
2299     list_entry_t *i;
2300
2301     for (i = transition_pages.next; i != &transition_pages; i = i->next)
2302     {
2303         *entry = CONTAINER_OF(i, page_store_entry_t, link);
2304
2305         if ((*entry)->address_space == block->address_space
2306             && PAGE_ALIGN((dword_t)(*entry)->address) == PAGE_ALIGN((dword_t)address))
2307         {
2308             return TRUE;
2309         }
2310     }
2311
2312     for (i = page_stores.next; i != &page_stores; i = i->next)
2313     {
2314         list_entry_t *j;
2315         *store = CONTAINER_OF(i, page_store_t, link);
2316
2317         for (j = (*store)->entry_list.next; j != &(*store)->entry_list; j = j->next)
2318         {
2319             *entry = CONTAINER_OF(j, page_store_entry_t, link);
2320
2321             if ((*entry)->address_space == block->address_space
2322                 && PAGE_ALIGN((dword_t)(*entry)->address) == PAGE_ALIGN((dword_t)address))
2323             {
2324                 return TRUE;
2325             }
2326         }
2327     }
2328
2329     return FALSE;
2330 }
2331
2332 bool_t memory_fault_handler(void *address, registers_t *regs)
2333 {
2334     int i;
2335     page_error_t problem;
2336     dword_t aligned_address = PAGE_ALIGN((dword_t)address);
2337     dword_t pd_index = ADDR_TO_PDE((dword_t)address);
2338     dword_t pt_index = ADDR_TO_PTE((dword_t)address);
2339     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
2340     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
2341     process_t *proc = get_current_process();
2342
2343     memory_address_space_t *address_space = (proc != NULL && check_usermode(address, 1))
2344                                             ? &proc->memory_space : &kernel_address_space;
2345     memory_block_t *block = find_block_by_addr(address_space, address);
2346     if (block == NULL) return FALSE;
2347
2348     if (!(regs->error_code & PAGE_ERROR_PRESENT_FLAG))
2349     {
2350         problem = PAGE_ERROR_NOTPRESENT;
2351     }
2352     else if (!(block->flags & MEMORY_BLOCK_USERMODE)
2353         && (regs->error_code & PAGE_ERROR_USERMODE_FLAG))
2354     {
2355         problem = PAGE_ERROR_UNPRIVILEGED;
2356     }
2357     else if (regs->error_code & PAGE_ERROR_WRITE_FLAG)
2358     {
2359         problem = PAGE_ERROR_READONLY;
2360     }
2361     else
2362     {
2363         KERNEL_CRASH_WITH_REGS("Unknown paging problem", regs);
2364     }
2365
2366     if ((block->flags & MEMORY_BLOCK_ACCESSIBLE) && (problem == PAGE_ERROR_NOTPRESENT))
2367     {
2368         page_store_t *store = NULL;
2369         page_store_entry_t *entry = NULL;
2370         byte_t buffer[PAGE_SIZE];
2371         dword_t bytes_read;
2372         dword_t page_flags = 0;
2373
2374         if (find_evicted_page(block, address, &store, &entry))
2375         {
2376             if (block->flags & MEMORY_BLOCK_ACCESSIBLE) page_flags |= PAGE_PRESENT;
2377             if ((block->flags & (MEMORY_BLOCK_WRITABLE | MEMORY_BLOCK_COPY_ON_WRITE))
2378                 == MEMORY_BLOCK_WRITABLE)
2379             {
2380                 page_flags |= PAGE_WRITABLE;
2381             }
2382
2383             if (block->flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
2384             else page_flags |= PAGE_GLOBAL;
2385
2386             if (entry->number != INVALID_STORE_NUMBER)
2387             {
2388                 enable_ints();
2389                 dword_t ret = syscall_read_file(store->file_handle, buffer, (qword_t)entry->number * (qword_t)PAGE_SIZE, PAGE_SIZE, &bytes_read);
2390                 disable_ints();
2391
2392                 if ((page_directory[pd_index] & PAGE_PRESENT) && (page_table[pt_index] & PAGE_PRESENT))
2393                 {
2394                     return TRUE;
2395                 }
2396
2397                 if (ret != ERR_SUCCESS) return FALSE;
2398
2399                 ret = alloc_page((void*)aligned_address, page_flags);
2400                 if (ret != ERR_SUCCESS) return FALSE;
2401
2402                 list_entry_t *ptr;
2403                 for (ptr = store->entry_list.next; ptr != &store->entry_list; ptr = ptr->next)
2404                 {
2405                     page_store_entry_t *other_entry = CONTAINER_OF(ptr, page_store_entry_t, link);
2406
2407                     if (entry != other_entry && other_entry->number == entry->number)
2408                     {
2409                         list_remove(&other_entry->link);
2410                         list_append(&transition_pages, &other_entry->link);
2411
2412                         other_entry->physical = get_physical_address((void*)aligned_address);
2413                         other_entry->number = INVALID_STORE_NUMBER;
2414                     }
2415                 }
2416
2417                 clear_bit(store->bitmap, entry->number);
2418                 list_remove(&entry->link);
2419                 free(entry);
2420
2421                 memcpy((void*)aligned_address, buffer, PAGE_SIZE);
2422                 address_space->stats.evicted -= PAGE_SIZE;
2423                 return TRUE;
2424             }
2425             else
2426             {
2427                 if (map_page(entry->physical, entry->address, page_flags) == ERR_SUCCESS)
2428                 {
2429                     list_remove(&entry->link);
2430                     free(entry);
2431                     address_space->stats.evicted -= PAGE_SIZE;
2432                     return TRUE;
2433                 }
2434             }
2435
2436             return FALSE;
2437         }
2438         else
2439         {
2440             list_entry_t *ptr;
2441             shared_page_t *page = NULL;
2442             qword_t offset = block->section_offset + (qword_t)aligned_address - (qword_t)block->address;
2443
2444             page_flags = PAGE_PRESENT;
2445             if (block->flags & MEMORY_BLOCK_WRITABLE) page_flags |= PAGE_WRITABLE;
2446
2447             if (block->flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
2448             else page_flags |= PAGE_GLOBAL;
2449
2450             if (block->section && offset < (qword_t)block->section->size)
2451             {
2452                 ASSERT(PAGE_OFFSET(offset) == 0);
2453
2454                 for (ptr = block->section->page_list.next; ptr != &block->section->page_list; ptr = ptr->next)
2455                 {
2456                     page = CONTAINER_OF(ptr, shared_page_t, link);
2457                     if (page->offset == offset) break;
2458                 }
2459
2460                 if (ptr != &block->section->page_list)
2461                 {
2462                     return (map_page(page->physical, (void*)aligned_address, page_flags) == ERR_SUCCESS);
2463                 }
2464             }
2465
2466             memset(buffer, 0, PAGE_SIZE);
2467
2468             if (block->section && block->section->file && offset < (qword_t)block->section->size)
2469             {
2470                 enable_ints();
2471                 file_instance_t *file = block->section->file;
2472                 acquire_resource_shared(&file->global->volume->resource);
2473                 dword_t ret = file->global->volume->driver->read_file(file, buffer, offset, PAGE_SIZE, &bytes_read);
2474                 release_resource(&file->global->volume->resource);
2475                 disable_ints();
2476                 if (ret != ERR_SUCCESS && ret != ERR_BEYOND) return FALSE;
2477             }
2478
2479             dword_t ret = alloc_page((void*)aligned_address, page_flags | PAGE_WRITABLE);
2480             if (ret != ERR_SUCCESS) return FALSE;
2481
2482             memcpy((void*)aligned_address, buffer, PAGE_SIZE);
2483             set_page_flags((void*)aligned_address, page_flags);
2484
2485             if (block->section && offset < (qword_t)block->section->size)
2486             {
2487                 page = (shared_page_t*)malloc(sizeof(shared_page_t));
2488                 if (page == NULL)
2489                 {
2490                     free_page((void*)aligned_address);
2491                     return FALSE;
2492                 }
2493
2494                 page->physical = get_physical_address((void*)aligned_address);
2495                 page->offset = offset;
2496
2497                 list_append(&block->section->page_list, &page->link);
2498             }
2499
2500             address_space->stats.committed += PAGE_SIZE;
2501             return TRUE;
2502         }
2503     }
2504
2505     if ((block->flags & (MEMORY_BLOCK_COPY_ON_WRITE | MEMORY_BLOCK_WRITABLE))
2506         == (MEMORY_BLOCK_COPY_ON_WRITE | MEMORY_BLOCK_WRITABLE)
2507         && (problem == PAGE_ERROR_READONLY))
2508     {
2509         if (!(page_directory[pd_index] & PAGE_WRITABLE))
2510         {
2511             void *table_phys = (void*)PAGE_ALIGN(page_directory[pd_index]);
2512
2513             if (get_page(table_phys)->ref_count > 1)
2514             {
2515                 void *table_copy = alloc_physical_page();
2516                 if (table_copy == NULL) return FALSE;
2517
2518                 dword_t *temporary = map_temporary_page(table_copy, PAGE_PRESENT | PAGE_WRITABLE);
2519                 if (temporary == NULL)
2520                 {
2521                     free_physical_page(table_copy);
2522                     return FALSE;
2523                 }
2524
2525                 for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++)
2526                 {
2527                     if (page_table[i])
2528                     {
2529                         reference_page((void*)PAGE_ALIGN(page_table[i]));
2530                         temporary[i] = page_table[i] & ~PAGE_WRITABLE;
2531                     }
2532                 }
2533
2534                 unmap_temporary_page(temporary);
2535
2536                 reference_page(table_copy);
2537                 dereference_page(table_phys);
2538
2539                 page_directory[pd_index] = PAGE_ALIGN((dword_t)table_copy)
2540                                            | PAGE_OFFSET(page_directory[pd_index])
2541                                            | PAGE_WRITABLE;
2542                 invalidate_tlb(page_table);
2543             }
2544             else
2545             {
2546                 page_directory[pd_index] |= PAGE_WRITABLE;
2547                 invalidate_tlb(page_table);
2548
2549                 for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++)
2550                 {
2551                     page_table[i] &= ~PAGE_WRITABLE;
2552                     invalidate_tlb((void*)((pd_index << 22) | (i << 12)));
2553                 }
2554             }
2555         }
2556
2557         if (!(page_table[pt_index] & PAGE_WRITABLE))
2558         {
2559             void *phys = (void*)PAGE_ALIGN(page_table[pt_index]);
2560
2561             if (get_page(phys)->ref_count > 1)
2562             {
2563                 void *page_copy = alloc_physical_page();
2564                 if (page_copy == NULL) return FALSE;
2565
2566                 write_physical(page_copy, (void*)PAGE_ALIGN((dword_t)address), PAGE_SIZE);
2567                 reference_page(page_copy);
2568                 dereference_page(phys);
2569
2570                 page_table[pt_index] = PAGE_ALIGN((dword_t)page_copy)
2571                                        | PAGE_OFFSET(page_table[pt_index])
2572                                        | PAGE_WRITABLE;
2573                 invalidate_tlb((void*)aligned_address);
2574             }
2575             else
2576             {
2577                 page_table[pt_index] |= PAGE_WRITABLE;
2578                 invalidate_tlb((void*)aligned_address);
2579             }
2580         }
2581
2582         return TRUE;
2583     }
2584
2585     return FALSE;
2586 }
2587
2588 void memory_init(multiboot_mmap_t *mmap_addr, dword_t mmap_length)
2589 {
2590     dword_t i, j;
2591     multiboot_mmap_t *mmap = mmap_addr;
2592     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
2593
2594     fix_overlapping_sections(mmap_addr, mmap_length);
2595
2596     puts("\nMemory map:\n\nBase\t\t\tLength\t\t\tType");
2597     puts("------------------------------------------------------------");
2598
2599     while ((dword_t)mmap < (dword_t)mmap_addr + mmap_length)
2600     {
2601         printf("0x%08X%08X\t0x%08X%08X\t%s\n",
2602                mmap->base_high,
2603                mmap->base_low,
2604                mmap->length_high,
2605                mmap->length_low,
2606                (mmap->type == 1) ? "Usable" : "Not Usable");
2607
2608         if (mmap->type == 1
2609             && mmap->base_high == 0
2610             && mmap->length_high == 0
2611             && mmap->length_low < (0xFFFFFFFF - mmap->base_low)
2612             && mmap->length_low > 0)
2613         {
2614             dword_t start_addr = mmap->base_low;
2615             if (start_addr < MEM_FIRST_PHYS_ADDR) start_addr = MEM_FIRST_PHYS_ADDR;
2616             start_addr = PAGE_ALIGN_UP(start_addr);
2617             dword_t end_addr = PAGE_ALIGN_UP(mmap->base_low + mmap->length_low);
2618             dword_t page = end_addr - PAGE_SIZE;
2619
2620             while (page >= start_addr)
2621             {
2622                 dword_t stack_address = (dword_t)&physical_memory_stack[num_free_pages];
2623                 dword_t pd_index = ADDR_TO_PDE(stack_address);
2624                 dword_t pt_index = ADDR_TO_PTE(stack_address);
2625                 dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + pd_index * PAGE_SIZE);
2626
2627                 if (!(page_directory[pd_index] & PAGE_PRESENT))
2628                 {
2629                     page_directory[pd_index] = start_addr | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2630                     start_addr += PAGE_SIZE;
2631                     invalidate_tlb(page_table);
2632                     memset(page_table, 0, PAGE_SIZE);
2633                     total_physical_pages++;
2634                     continue;
2635                 }
2636
2637                 if (!(page_table[pt_index] & PAGE_PRESENT))
2638                 {
2639                     page_table[pt_index] = start_addr | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2640                     start_addr += PAGE_SIZE;
2641                     invalidate_tlb((void*)stack_address);
2642                     total_physical_pages++;
2643                     continue;
2644                 }
2645
2646                 free_physical_page((void*)page);
2647                 page -= PAGE_SIZE;
2648             }
2649         }
2650
2651         mmap = (multiboot_mmap_t*)((dword_t)mmap + mmap->size + sizeof(dword_t));
2652     }
2653
2654     puts("------------------------------------------------------------");
2655     total_physical_pages += num_free_pages;
2656     pages = (page_t*)(KERNEL_POOL_START - total_physical_pages * sizeof(page_t));
2657
2658     for (i = PAGE_ALIGN((uintptr_t)pages); i < KERNEL_POOL_START; i += PAGE_SIZE)
2659     {
2660         dword_t pd_index = ADDR_TO_PDE(i);
2661         dword_t pt_index = ADDR_TO_PTE(i);
2662         dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + pd_index * PAGE_SIZE);
2663
2664         if (!(page_directory[pd_index] & PAGE_PRESENT))
2665         {
2666             page_directory[pd_index] = (uintptr_t)alloc_physical_page() | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2667             invalidate_tlb(page_table);
2668             memset(page_table, 0, PAGE_SIZE);
2669         }
2670
2671         if (!(page_table[pt_index] & PAGE_PRESENT))
2672         {
2673             page_table[pt_index] = (uintptr_t)alloc_physical_page() | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2674             invalidate_tlb((void*)i);
2675         }
2676     }
2677
2678     dword_t pages_inserted = 0;
2679
2680     for (i = 0; i < num_free_pages; i++)
2681     {
2682         pages[pages_inserted].phys_addr = PAGE_ALIGN((dword_t)physical_memory_stack[i]);
2683         pages[pages_inserted].ref_count = 0;
2684         pages_inserted++;
2685     }
2686
2687     for (i = KERNEL_PAGE_START; i <= KERNEL_PAGE_END; i++)
2688     {
2689         dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + i * PAGE_SIZE);
2690         if (!(page_directory[i] & PAGE_PRESENT)) continue;
2691
2692         for (j = 0; j < PAGE_SIZE / sizeof(dword_t); j++)
2693         {
2694             if (PAGE_ALIGN(page_table[j]) < MEM_FIRST_PHYS_ADDR) continue;
2695
2696             if (page_table[j] & PAGE_PRESENT)
2697             {
2698                 pages[pages_inserted].phys_addr = PAGE_ALIGN((dword_t)page_table[j]);
2699                 pages[pages_inserted].ref_count = 0;
2700                 pages_inserted++;
2701             }
2702         }
2703     }
2704
2705     ASSERT(pages_inserted == total_physical_pages);
2706     qsort(pages, total_physical_pages, sizeof(page_t), compare_page);
2707
2708     init_semaphore(&temporary_page_semaphore, TEMPORARY_PAGES, TEMPORARY_PAGES);
2709
2710     if (create_address_space((void*)KERNEL_POOL_START,
2711                              (KERNEL_POOL_END - KERNEL_POOL_START + PAGE_SIZE - 1) / PAGE_SIZE,
2712                              &kernel_address_space) != ERR_SUCCESS)
2713     {
2714         KERNEL_CRASH("Unable to create kernel address space");
2715     }
2716
2717     if (create_address_space((void*)MAPPING_START,
2718                              (MAPPING_END - MAPPING_START + PAGE_SIZE - 1) / PAGE_SIZE,
2719                              &mapping_space) != ERR_SUCCESS)
2720     {
2721         KERNEL_CRASH("Unable to create mapping space");
2722     }
2723
2724     set_page_directory((void*)PAGE_ALIGN(page_directory[PAGEDIR_SELF_ENTRY]));
2725
2726     for (i = KERNEL_PAGE_START; i <= KERNEL_PAGE_END; i++)
2727     {
2728         dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + i * PAGE_SIZE);
2729         if (!(page_directory[i] & PAGE_PRESENT)) continue;
2730
2731         for (j = 0; j < PAGE_SIZE / sizeof(dword_t); j++)
2732         {
2733             if (page_table[j] & PAGE_PRESENT) reference_page((void*)PAGE_ALIGN(page_table[j]));
2734         }
2735     }
2736
2737     for (i = USER_PAGE_START; i <= USER_PAGE_END; i++) page_directory[i] = 0;
2738     set_page_directory(get_page_directory());
2739
2740     if (cpu_features[0] & CPU_FEATURE_PGE)
2741     {
2742         asm volatile ("movl %cr4, %eax\n"
2743                       "orl $0x80, %eax\n"
2744                       "movl %eax, %cr4\n");
2745     }
2746 }