4d0798ac23b1819412aaa9dbdb58ec7be38f007b
[monolithium.git] / kernel / src / memory / memory.c
1 /*
2  * memory.c
3  *
4  * Copyright (C) 2016 Aleksandar Andrejevic <theflash@sdf.lonestar.org>
5  *
6  * This program is free software: you can redistribute it and/or modify
7  * it under the terms of the GNU Affero General Public License as
8  * published by the Free Software Foundation, either version 3 of the
9  * License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU Affero General Public License for more details.
15  *
16  * You should have received a copy of the GNU Affero General Public License
17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19
20 #include <memory.h>
21 #include <log.h>
22 #include <exception.h>
23 #include <process.h>
24 #include <syscalls.h>
25 #include <heap.h>
26 #include <cpu.h>
27 #include <semaphore.h>
28
29 static void **physical_memory_stack = (void**)MEM_STACK_VIRT_ADDR;
30 static DECLARE_LOCK(phys_mem_stack_lock);
31 static page_t *pages = NULL;
32 static void *current_page_directory = INVALID_PAGE;
33 static memory_address_space_t kernel_address_space;
34 static memory_address_space_t mapping_space;
35 static list_entry_t user_address_spaces = { &user_address_spaces, &user_address_spaces };
36 static dword_t total_physical_pages = 0;
37 static dword_t num_free_pages = 0;
38 static dword_t mem_tree_bitmap[TOTAL_PAGES / 32];
39 static DECLARE_LOCK(mem_tree_lock);
40 static semaphore_t temporary_page_semaphore;
41 static bool_t evicting = FALSE;
42 static DECLARE_LIST(transition_pages);
43 static DECLARE_LIST(page_stores);
44 static DECLARE_LOCK(page_store_lock);
45
46 static void *evict_page(void);
47
48 static inline void *alloc_physical_page(void)
49 {
50     void *page = INVALID_PAGE;
51
52     if (!evicting && num_free_pages <= EVICTION_THRESHOLD)
53     {
54         evicting = TRUE;
55         page = evict_page();
56         evicting = FALSE;
57
58         if (page != INVALID_PAGE) return page;
59     }
60
61     lock_acquire(&phys_mem_stack_lock);
62     if (num_free_pages) page = physical_memory_stack[--num_free_pages];
63     lock_release(&phys_mem_stack_lock);
64
65     return page;
66 }
67
68 static inline void free_physical_page(void *address)
69 {
70     lock_acquire(&phys_mem_stack_lock);
71     physical_memory_stack[num_free_pages++] = address;
72     lock_release(&phys_mem_stack_lock);
73 }
74
75 static int compare_page(const void *a, const void *b)
76 {
77     const page_t *page_a = (const page_t*)a;
78     const page_t *page_b = (const page_t*)b;
79
80     if (page_a->phys_addr < page_b->phys_addr) return -1;
81     else if (page_a->phys_addr > page_b->phys_addr) return 1;
82     else return 0;
83 }
84
85 static page_t *get_page(void *physical)
86 {
87     page_t key = { .phys_addr = (uintptr_t)physical };
88     if (pages == NULL) return NULL;
89     return (page_t*)bsearch(&key, pages, total_physical_pages, sizeof(page_t), compare_page);
90 }
91
92 static inline dword_t reference_page(void *physical)
93 {
94     page_t *page = get_page(physical);
95     if (!page) return 0;
96
97     return ++page->ref_count;
98 }
99
100 static inline dword_t dereference_page(void *physical)
101 {
102     page_t *page = get_page(physical);
103     if (!page) return 0;
104
105     return --page->ref_count;
106 }
107
108 static dword_t map_page(void *physical, void *virtual, dword_t flags)
109 {
110     dword_t i;
111     dword_t ret = ERR_SUCCESS;
112     critical_t critical;
113     dword_t phys_addr = PAGE_ALIGN((dword_t)physical);
114     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
115     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
116     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
117     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
118
119     flags &= 0x00000FFF;
120     enter_critical(&critical);
121
122     if (!(page_directory[pd_index] & PAGE_PRESENT))
123     {
124         void *table_page = alloc_physical_page();
125         if (table_page == INVALID_PAGE)
126         {
127             ret = ERR_NOMEMORY;
128             goto done;
129         }
130
131         reference_page(table_page);
132         page_directory[pd_index] = (dword_t)table_page | PAGE_PRESENT | PAGE_WRITABLE;
133
134         cpu_invalidate_tlb(page_table);
135         for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++) page_table[i] = 0;
136     }
137
138     page_directory[pd_index] |= flags;
139     if (page_table[pt_index] & PAGE_PRESENT)
140     {
141         ret = ERR_EXISTS;
142         goto done;
143     }
144
145     reference_page((void*)phys_addr);
146     page_table[pt_index] = phys_addr | flags | PAGE_PRESENT;
147     cpu_invalidate_tlb(virtual);
148
149 done:
150     leave_critical(&critical);
151     return ret;
152 }
153
154 static dword_t unmap_page(void *virtual)
155 {
156     dword_t i, ret = ERR_SUCCESS;
157     critical_t critical;
158     bool_t empty_dir = TRUE;
159     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
160     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
161     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
162     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
163
164     enter_critical(&critical);
165
166     if (!(page_directory[pd_index] & PAGE_PRESENT))
167     {
168         ret = ERR_NOTFOUND;
169         goto done;
170     }
171
172     if (!(page_table[pt_index] & PAGE_PRESENT))
173     {
174         ret = ERR_NOTFOUND;
175         goto done;
176     }
177
178     dereference_page((void*)PAGE_ALIGN(page_table[pt_index]));
179     page_table[pt_index] = 0;
180     cpu_invalidate_tlb((dword_t*)virt_addr);
181
182     for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++) if (page_table[i])
183     {
184         empty_dir = FALSE;
185         break;
186     }
187
188     if (empty_dir)
189     {
190         void *table_page = (void*)PAGE_ALIGN(page_directory[pd_index]);
191         page_directory[pd_index] = 0;
192         cpu_invalidate_tlb(page_table);
193
194         if (dereference_page(table_page) == 0)
195         {
196             free_physical_page(table_page);
197         }
198     }
199
200 done:
201     leave_critical(&critical);
202     return ret;
203 }
204
205 static dword_t get_page_flags(void *virtual)
206 {
207     dword_t virt_addr = PAGE_ALIGN((uintptr_t)virtual);
208     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
209     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
210     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
211
212     if (!(page_directory[pd_index] & PAGE_PRESENT)) return 0;
213     if (!(page_table[pt_index] & PAGE_PRESENT)) return 0;
214
215     return PAGE_OFFSET(page_table[pt_index]);
216 }
217
218 static dword_t set_page_flags(void *virtual, dword_t flags)
219 {
220     dword_t ret = ERR_SUCCESS;
221     critical_t critical;
222     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
223     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
224     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
225     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
226
227     flags &= 0x00000FFF;
228     enter_critical(&critical);
229
230     if (!(page_directory[pd_index] & PAGE_PRESENT))
231     {
232         ret = ERR_NOTFOUND;
233         goto done;
234     }
235
236     if (!(page_table[pt_index] & PAGE_PRESENT))
237     {
238         ret = ERR_NOTFOUND;
239         goto done;
240     }
241
242     page_directory[pd_index] |= flags;
243     page_table[pt_index] = PAGE_ALIGN(page_table[pt_index]) | flags | PAGE_PRESENT;
244     cpu_invalidate_tlb((void*)virt_addr);
245
246 done:
247     leave_critical(&critical);
248     return ret;
249 }
250
251 static void *map_temporary_page(void *physical, dword_t flags)
252 {
253     int i;
254     wait_semaphore(&temporary_page_semaphore, 1, NO_TIMEOUT);
255
256     for (i = TEMPORARY_PAGES - 1; i >= temporary_page_semaphore.count ; i--)
257     {
258         void *address = (void*)(TEMPORARY_ADDR + i * PAGE_SIZE);
259
260         if (get_physical_address(address) == INVALID_PAGE)
261         {
262             if (map_page(physical, address, flags) == ERR_SUCCESS) return address;
263             break;
264         }
265     }
266
267     return NULL;
268 }
269
270 static void unmap_temporary_page(void *virtual)
271 {
272     unmap_page(virtual);
273     release_semaphore(&temporary_page_semaphore, 1);
274 }
275
276 static inline dword_t alloc_page(void *virtual, dword_t flags)
277 {
278     void *phys = alloc_physical_page();
279     if (phys == INVALID_PAGE) return ERR_NOMEMORY;
280
281     dword_t ret = map_page(phys, virtual, flags);
282     if (ret != ERR_SUCCESS) free_physical_page(phys);
283
284     return ret;
285 }
286
287 static inline dword_t free_page(void *virtual)
288 {
289     void *phys = get_physical_address(virtual);
290     if (phys == INVALID_PAGE) return ERR_INVALID;
291
292     unmap_page(virtual);
293
294     page_t *page = get_page(phys);
295     if (page != NULL && page->ref_count > 0) free_physical_page(phys);
296
297     return ERR_SUCCESS;
298 }
299
300 static void *evict_page_from_address_space(memory_address_space_t *space)
301 {
302     void *physical = INVALID_PAGE;
303     int chances = 2;
304     dword_t cached_directory[PAGE_SIZE / sizeof(dword_t)];
305     dword_t *table = NULL;
306
307     if (read_physical(space->page_directory, cached_directory, PAGE_SIZE) != ERR_SUCCESS)
308     {
309         return INVALID_PAGE;
310     }
311
312     if (!space->evict_blk_ptr) space->evict_blk_ptr = space->evictable_blocks.next;
313     memory_block_t *block = CONTAINER_OF(space->evict_blk_ptr, memory_block_t, evict_link);
314     dword_t prev_pd_index = (dword_t)-1;
315     dword_t address;
316     dword_t pd_index, pt_index;
317
318     while (chances)
319     {
320         address = (dword_t)block->address + space->evict_page_num * PAGE_SIZE;
321         pd_index = ADDR_TO_PDE(address);
322         pt_index = ADDR_TO_PTE(address);
323         if (!(cached_directory[pd_index] & PAGE_PRESENT)) goto next;
324
325         if (prev_pd_index != pd_index)
326         {
327             if (table) unmap_temporary_page(table);
328             table = map_temporary_page((void*)PAGE_ALIGN(cached_directory[pd_index]),
329                                        PAGE_PRESENT | PAGE_WRITABLE);
330             if (table == NULL) break;
331             prev_pd_index = pd_index;
332         }
333
334         if (table[pt_index])
335         {
336             if (!(table[pt_index] & PAGE_ACCESSED))
337             {
338                 physical = (void*)PAGE_ALIGN(table[pt_index]);
339                 break;
340             }
341
342             table[pt_index] &= ~PAGE_ACCESSED;
343         }
344
345 next:
346         space->evict_page_num++;
347
348         if (space->evict_page_num == (dword_t)block->size)
349         {
350             space->evict_page_num = 0;
351             space->evict_blk_ptr = space->evict_blk_ptr->next;
352
353             if (space->evict_blk_ptr == &space->evictable_blocks)
354             {
355                 space->evict_blk_ptr = space->evict_blk_ptr->next;
356                 chances--;
357             }
358
359             if (space->evict_blk_ptr == &space->evictable_blocks) break;
360             block = CONTAINER_OF(space->evict_blk_ptr, memory_block_t, evict_link);
361         }
362     }
363
364     if (physical == INVALID_PAGE) goto cleanup;
365
366     dword_t i;
367     list_entry_t *ptr;
368     page_store_t *store = NULL;
369     byte_t buffer[PAGE_SIZE];
370
371     dword_t ret = read_physical(physical, buffer, PAGE_SIZE);
372     if (ret != ERR_SUCCESS)
373     {
374         physical = INVALID_PAGE;
375         goto cleanup;
376     }
377
378     for (ptr = page_stores.next; ptr != &page_stores; ptr = ptr->next)
379     {
380         store = CONTAINER_OF(ptr, page_store_t, link);
381
382         for (i = 0; i < store->max_entries; i++) if (!test_bit(store->bitmap, i)) break;
383         if (i == store->max_entries) continue;
384     }
385
386     if (ptr == &page_stores)
387     {
388         physical = INVALID_PAGE;
389         goto cleanup;
390     }
391
392     page_store_entry_t *entry = (page_store_entry_t*)malloc(sizeof(page_store_entry_t));
393     if (entry == NULL)
394     {
395         physical = INVALID_PAGE;
396         goto cleanup;
397     }
398
399     space->stats.evicted += PAGE_SIZE;
400     entry->address = (void*)address;
401     entry->address_space = space;
402     entry->number = INVALID_STORE_NUMBER;
403     entry->physical = INVALID_PAGE;
404
405     if (dereference_page(physical) == 0)
406     {
407         entry->number = i;
408
409         dword_t bytes_written;
410         ret = syscall_write_file(store->file_handle, buffer, (qword_t)entry->number * (qword_t)PAGE_SIZE, PAGE_SIZE, &bytes_written);
411         if (ret != ERR_SUCCESS)
412         {
413             reference_page(physical);
414             free(entry);
415             physical = INVALID_PAGE;
416             goto cleanup;
417         }
418
419         set_bit(store->bitmap, i);
420         list_append(&store->entry_list, &entry->link);
421
422         for (ptr = transition_pages.next; ptr != &transition_pages; ptr = ptr->next)
423         {
424             page_store_entry_t *other_entry = CONTAINER_OF(ptr, page_store_entry_t, link);
425
426             if (other_entry->physical == physical)
427             {
428                 ASSERT(other_entry->number == INVALID_STORE_NUMBER);
429
430                 list_remove(&other_entry->link);
431                 list_append(&store->entry_list, &other_entry->link);
432
433                 other_entry->number = entry->number;
434                 other_entry->physical = INVALID_PAGE;
435             }
436         }
437     }
438     else
439     {
440         entry->physical = physical;
441         list_append(&transition_pages, &entry->link);
442         physical = INVALID_PAGE;
443     }
444
445     table[pt_index] = 0;
446     if (space->page_directory == get_page_directory()) cpu_invalidate_tlb((void*)address);
447
448 cleanup:
449     if (table) unmap_temporary_page(table);
450     return physical;
451 }
452
453 static void *evict_page(void)
454 {
455     if (pages == NULL) return INVALID_PAGE;
456
457     list_entry_t *ptr;
458
459     for (ptr = user_address_spaces.next; ptr != &user_address_spaces; ptr = ptr->next)
460     {
461         memory_address_space_t *space = CONTAINER_OF(ptr, memory_address_space_t, link);
462         void *page = evict_page_from_address_space(space);
463         if (page != INVALID_PAGE) return page;
464     }
465
466     return evict_page_from_address_space(&kernel_address_space);
467 }
468
469 static memory_block_t *mem_tree_alloc(void)
470 {
471     dword_t i;
472     memory_block_t *block = NULL;
473
474     lock_acquire(&mem_tree_lock);
475     for (i = 0; i < TOTAL_PAGES; i++) if (!test_bit(mem_tree_bitmap, i)) break;
476
477     if (i < TOTAL_PAGES)
478     {
479         block = (memory_block_t*)(MEM_TREE_BLOCKS + i * sizeof(memory_block_t));
480
481         if ((get_physical_address(block) != INVALID_PAGE)
482             || (alloc_page(block, PAGE_GLOBAL | PAGE_WRITABLE | PAGE_PRESENT) == ERR_SUCCESS))
483         {
484             set_bit(mem_tree_bitmap, i);
485         }
486         else
487         {
488             block = NULL;
489         }
490     }
491
492     lock_release(&mem_tree_lock);
493     return block;
494 }
495
496 static void mem_tree_free(memory_block_t *block)
497 {
498     dword_t index = ((dword_t)block - MEM_TREE_BLOCKS) / sizeof(memory_block_t);
499     bool_t busy = FALSE;
500     dword_t i, page = PAGE_ALIGN((dword_t)block);
501
502     lock_acquire(&mem_tree_lock);
503     clear_bit(mem_tree_bitmap, index);
504
505     for (i = page; i < page + PAGE_SIZE; i += sizeof(memory_block_t))
506     {
507         index = (i - MEM_TREE_BLOCKS) / sizeof(memory_block_t);
508         if (test_bit(mem_tree_bitmap, index))
509         {
510             busy = TRUE;
511             break;
512         }
513     }
514
515     if (!busy) free_page((void*)page);
516     lock_release(&mem_tree_lock);
517 }
518
519 static memory_block_t *find_block_by_addr_internal(memory_block_t *block, void *address)
520 {
521     qword_t key = (qword_t)(dword_t)address;
522     qword_t start_addr = block->address;
523     qword_t end_addr = start_addr + block->size * PAGE_SIZE;
524
525     if (key >= start_addr && key < end_addr) return block;
526
527     if (key < start_addr)
528     {
529         if (!block->by_addr_node.left) return NULL;
530
531         memory_block_t *left_block = CONTAINER_OF(block->by_addr_node.left, memory_block_t, by_addr_node);
532         return find_block_by_addr_internal(left_block, address);
533     }
534     else
535     {
536         if (!block->by_addr_node.right) return NULL;
537
538         memory_block_t *right_block = CONTAINER_OF(block->by_addr_node.right, memory_block_t, by_addr_node);
539         return find_block_by_addr_internal(right_block, address);
540     }
541 }
542
543 static memory_block_t *find_block_by_addr(memory_address_space_t *space, void *address)
544 {
545     if (!space->by_addr_tree.root) return NULL;
546     memory_block_t *root = CONTAINER_OF(space->by_addr_tree.root, memory_block_t, by_addr_node);
547     return find_block_by_addr_internal(root, address);
548 }
549
550 static bool_t clone_blocks_recursive(memory_address_space_t *space, memory_block_t *block)
551 {
552     memory_block_t *clone = mem_tree_alloc();
553     if (clone == NULL) return FALSE;
554
555     clone->address = block->address;
556     clone->size = block->size;
557     block->flags |= MEMORY_BLOCK_COPY_ON_WRITE;
558     clone->flags = block->flags;
559     clone->address_space = space;
560     clone->section = block->section;
561
562     avl_tree_insert(&space->by_addr_tree, &clone->by_addr_node);
563     avl_tree_insert(&space->by_size_tree, &clone->by_size_node);
564
565     memory_block_t *left_block = CONTAINER_OF(block->by_addr_node.left, memory_block_t, by_addr_node);
566     memory_block_t *right_block = CONTAINER_OF(block->by_addr_node.right, memory_block_t, by_addr_node);
567
568     if ((block->by_addr_node.left && !clone_blocks_recursive(space, left_block))
569         || (block->by_addr_node.right && !clone_blocks_recursive(space, right_block)))
570     {
571         avl_tree_remove(&space->by_addr_tree, &clone->by_addr_node);
572         avl_tree_remove(&space->by_size_tree, &clone->by_size_node);
573         mem_tree_free(clone);
574         return FALSE;
575     }
576
577     return TRUE;
578 }
579
580 static inline void release_memory_block(memory_block_t *block)
581 {
582     dword_t page;
583     dword_t start_address = (dword_t)block->address;
584     dword_t end_address = start_address + (dword_t)block->size * PAGE_SIZE;
585
586     critical_t critical;
587     enter_critical(&critical);
588     void *old_page_dir = get_page_directory();
589     set_page_directory(block->address_space->page_directory);
590
591     for (page = start_address; page < end_address; page += PAGE_SIZE)
592     {
593         free_page((void*)page);
594     }
595
596     set_page_directory(old_page_dir);
597     leave_critical(&critical);
598
599     if (block->section)
600     {
601         dereference(&block->section->header);
602         block->section = NULL;
603     }
604
605     list_entry_t *i;
606
607     for (i = transition_pages.next; i != &transition_pages; i = i->next)
608     {
609         page_store_entry_t *entry = CONTAINER_OF(i, page_store_entry_t, link);
610
611         if (entry->address_space == block->address_space
612             && (dword_t)entry->address >= start_address
613             && ((dword_t)entry->address < end_address))
614         {
615             list_remove(&entry->link);
616             free(entry);
617         }
618     }
619
620     lock_acquire(&page_store_lock);
621
622     for (i = page_stores.next; i != &page_stores; i = i->next)
623     {
624         list_entry_t *j;
625         page_store_t *store = CONTAINER_OF(i, page_store_t, link);
626
627         for (j = store->entry_list.next; j != &store->entry_list; j = j->next)
628         {
629             page_store_entry_t *entry = CONTAINER_OF(j, page_store_entry_t, link);
630
631             if (entry->address_space == block->address_space
632                 && (dword_t)entry->address >= start_address
633                 && ((dword_t)entry->address < end_address))
634             {
635                 if (entry->number != INVALID_STORE_NUMBER) clear_bit(store->bitmap, entry->number);
636                 list_remove(&entry->link);
637                 free(entry);
638             }
639         }
640     }
641
642     lock_release(&page_store_lock);
643 }
644
645 static void free_blocks_recursive(memory_block_t *block)
646 {
647     release_memory_block(block);
648
649     if (block->by_addr_node.left)
650     {
651         memory_block_t *left_block = CONTAINER_OF(block->by_addr_node.left, memory_block_t, by_addr_node);
652         free_blocks_recursive(left_block);
653     }
654
655     if (block->by_addr_node.right)
656     {
657         memory_block_t *right_block = CONTAINER_OF(block->by_addr_node.right, memory_block_t, by_addr_node);
658         free_blocks_recursive(right_block);
659     }
660
661     mem_tree_free(block);
662 }
663
664 static memory_block_t *find_free_block_internal(memory_block_t *root, void *address, dword_t size)
665 {
666     avl_node_t *ptr;
667
668     if (root->by_size_node.left && (dword_t)root->size > size)
669     {
670         memory_block_t *left = CONTAINER_OF(root->by_size_node.left, memory_block_t, by_size_node);
671         memory_block_t *block = find_free_block_internal(left, address, size);
672         if (block) return block;
673     }
674
675     if ((dword_t)root->size >= size)
676     {
677         for (ptr = &root->by_size_node; ptr != NULL; ptr = ptr->next_equal)
678         {
679             memory_block_t *block = CONTAINER_OF(ptr, memory_block_t, by_size_node);
680
681             if (!(block->flags & MEMORY_BLOCK_FREE)) continue;
682
683             if (address != NULL)
684             {
685                 dword_t block_start = (dword_t)block->address;
686                 dword_t block_end = block_start + ((dword_t)block->size * PAGE_SIZE) - 1;
687
688                 dword_t needed_start = (dword_t)address;
689                 dword_t needed_end = needed_start + (size * PAGE_SIZE) - 1;
690
691                 if ((needed_start < block_start) || (needed_end > block_end)) continue;
692             }
693
694             return block;
695         }
696     }
697
698     if (!root->by_size_node.right) return NULL;
699     memory_block_t *right = CONTAINER_OF(root->by_size_node.right, memory_block_t, by_size_node);
700     return find_free_block_internal(right, address, size);
701 }
702
703 static memory_block_t *find_free_block(memory_address_space_t *address_space, void *address, dword_t size)
704 {
705     memory_block_t *root_block = CONTAINER_OF(address_space->by_size_tree.root, memory_block_t, by_size_node);
706     return find_free_block_internal(root_block, address, size);
707 }
708
709 static void *create_page_directory(void)
710 {
711     dword_t *current = (dword_t*)PAGE_DIRECTORY_ADDR;
712     dword_t new_dir_buffer[PAGE_SIZE / sizeof(dword_t)];
713
714     memset(&new_dir_buffer[USER_PAGE_START],
715            0,
716            (USER_PAGE_END - USER_PAGE_START + 1) * sizeof(dword_t));
717
718     memcpy(&new_dir_buffer[KERNEL_PAGE_START],
719            &current[KERNEL_PAGE_START],
720            (KERNEL_PAGE_END - KERNEL_PAGE_START + 1) * sizeof(dword_t));
721
722     void *directory = alloc_physical_page();
723     if (directory == NULL) return NULL;
724
725     new_dir_buffer[PAGEDIR_SELF_ENTRY] = (dword_t)directory | PAGE_PRESENT | PAGE_WRITABLE;
726     write_physical(directory, new_dir_buffer, PAGE_SIZE);
727
728     return directory;
729 }
730
731 static void fix_overlapping_sections(multiboot_tag_mmap_t *mmap)
732 {
733     multiboot_mmap_entry_t *entry;
734
735     for (entry = (multiboot_mmap_entry_t*)(mmap + 1);
736          (uintptr_t)entry < ((uintptr_t)mmap + mmap->size);
737          entry = (multiboot_mmap_entry_t*)((uintptr_t)entry + mmap->entry_size))
738     {
739         multiboot_mmap_entry_t *ptr;
740
741         for (ptr = (multiboot_mmap_entry_t*)(mmap + 1);
742              (uintptr_t)ptr < (uintptr_t)entry;
743              ptr = (multiboot_mmap_entry_t*)((uintptr_t)ptr + mmap->entry_size))
744         {
745             qword_t entry_end = entry->base + entry->length;
746             qword_t ptr_end = ptr->base + ptr->length;
747
748             if (entry->base > ptr->base && entry->base < ptr_end)
749             {
750                 entry->base = ptr_end;
751                 if (entry->base >= entry_end) entry->length = 0;
752                 else entry->length = entry_end - entry->base;
753             }
754             else if (ptr->base > entry->base && ptr->base < entry_end)
755             {
756                 ptr->base = entry_end;
757                 if (ptr->base >= ptr_end) ptr->length = 0;
758                 else ptr->length = ptr_end - ptr->base;
759             }
760         }
761     }
762 }
763
764 static inline memory_block_t *combine_blocks_forward(memory_block_t *mem_block)
765 {
766     while (TRUE)
767     {
768         avl_node_t *next = avl_get_next_node(&mem_block->by_addr_node);
769         if (!next) break;
770
771         memory_block_t *next_block = CONTAINER_OF(next, memory_block_t, by_addr_node);
772         if (!(next_block->flags & MEMORY_BLOCK_FREE)) break;
773
774         size_t new_size = mem_block->size + next_block->size;
775         avl_tree_change_key(&mem_block->address_space->by_size_tree, &mem_block->by_size_node, &new_size);
776
777         avl_tree_remove(&mem_block->address_space->by_addr_tree, &next_block->by_addr_node);
778         avl_tree_remove(&mem_block->address_space->by_size_tree, &next_block->by_size_node);
779         mem_tree_free(next_block);
780     }
781
782     return mem_block;
783 }
784
785 static inline memory_block_t *combine_blocks_backward(memory_block_t *mem_block)
786 {
787     while (TRUE)
788     {
789         avl_node_t *previous = avl_get_previous_node(&mem_block->by_addr_node);
790         if (!previous) break;
791
792         memory_block_t *prev_block = CONTAINER_OF(previous, memory_block_t, by_addr_node);
793         if (!(prev_block->flags & MEMORY_BLOCK_FREE)) break;
794
795         size_t new_size = prev_block->size + mem_block->size;
796         avl_tree_change_key(&mem_block->address_space->by_size_tree, &prev_block->by_size_node, &new_size);
797
798         avl_tree_remove(&mem_block->address_space->by_addr_tree, &mem_block->by_addr_node);
799         avl_tree_remove(&mem_block->address_space->by_size_tree, &mem_block->by_size_node);
800         mem_tree_free(mem_block);
801
802         mem_block = prev_block;
803     }
804
805     return mem_block;
806 }
807
808 void memory_cleanup(object_t *obj)
809 {
810     memory_section_t *section = (memory_section_t*)obj;
811     if (section->file) dereference(&section->file->header);
812 }
813
814 void *get_page_directory(void)
815 {
816     return current_page_directory;
817 }
818
819 void set_page_directory(void *phys_addr)
820 {
821     current_page_directory = phys_addr;
822     cpu_write_page_table_register((uintptr_t)phys_addr);
823 }
824
825 void *get_physical_address(void *virtual)
826 {
827     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
828     dword_t pd_index = ADDR_TO_PDE(virt_addr), pt_index = ADDR_TO_PTE(virt_addr);
829     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
830     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
831
832     if (!(page_directory[pd_index] & PAGE_PRESENT)) return INVALID_PAGE;
833     if (!(page_table[pt_index] & PAGE_PRESENT)) return INVALID_PAGE;
834
835     return (void*)(PAGE_ALIGN(page_table[pt_index]) + PAGE_OFFSET((dword_t)virtual));
836 }
837
838 dword_t map_memory_internal(void *physical, void *virtual, uintptr_t size, dword_t page_flags)
839 {
840     dword_t i, j;
841     dword_t phys_addr = PAGE_ALIGN((dword_t)physical);
842     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
843     size = PAGE_ALIGN_UP(size);
844     page_flags &= 0xFFF;
845
846     for (i = 0; i < size; i += PAGE_SIZE)
847     {
848         dword_t ret = map_page((void*)(phys_addr + i), (void*)(virt_addr + i), page_flags);
849         if (ret != ERR_SUCCESS)
850         {
851             for (j = 0; j < i; j += PAGE_SIZE) unmap_page((void*)(virt_addr + j));
852             return ret;
853         }
854     }
855
856     return ERR_SUCCESS;
857 }
858
859 void unmap_memory_internal(void *virtual, dword_t size)
860 {
861     dword_t i;
862     dword_t virt_addr = PAGE_ALIGN((dword_t)virtual);
863     size = PAGE_ALIGN_UP(size);
864
865     for (i = 0; i < size; i += PAGE_SIZE)
866     {
867         void *page_addr = (void*)(virt_addr + i);
868         void *physical = get_physical_address(page_addr);
869
870         unmap_page(page_addr);
871         dereference_page(physical);
872     }
873 }
874
875 dword_t map_memory_in_address_space(memory_address_space_t *address_space,
876                                     void *physical,
877                                     void **virtual,
878                                     uintptr_t size,
879                                     dword_t block_flags)
880 {
881     dword_t ret;
882     void *address = (void*)PAGE_ALIGN((uintptr_t)*virtual);
883     uintptr_t aligned_physical = PAGE_ALIGN((uintptr_t)physical);
884     if (*virtual != NULL && PAGE_OFFSET((uintptr_t)*virtual) != PAGE_OFFSET((uintptr_t)physical)) return ERR_INVALID;
885
886     size = (PAGE_ALIGN_UP((uintptr_t)physical + size - 1) - aligned_physical) >> 12;
887     lock_acquire(&address_space->lock);
888
889     memory_block_t *block = find_free_block(address_space, address, size);
890     if (block == NULL)
891     {
892         lock_release(&address_space->lock);
893         return ERR_NOMEMORY;
894     }
895
896     dword_t flags = PAGE_GLOBAL;
897     dword_t real_address = (address != NULL) ? (dword_t)address : (dword_t)block->address;
898
899     block_flags &= ~MEMORY_BLOCK_EVICTABLE;
900     if (block_flags & MEMORY_BLOCK_ACCESSIBLE) flags |= PAGE_PRESENT;
901     if (block_flags & MEMORY_BLOCK_WRITABLE) flags |= PAGE_WRITABLE;
902     if (block_flags & MEMORY_BLOCK_USERMODE) flags |= PAGE_USERMODE;
903
904     ret = map_memory_internal((void*)aligned_physical, (void*)real_address, size * PAGE_SIZE, flags);
905     if (ret != ERR_SUCCESS)
906     {
907         lock_release(&address_space->lock);
908         return ret;
909     }
910
911     if ((dword_t)block->address < (dword_t)address)
912     {
913         memory_block_t *new_block = mem_tree_alloc();
914         new_block->flags = MEMORY_BLOCK_FREE;
915         new_block->address = block->address;
916         new_block->size = (size_t)(((dword_t)address - block->address) / PAGE_SIZE);
917         new_block->address_space = address_space;
918         new_block->section = NULL;
919
920         size_t new_size = block->size - new_block->size;
921         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &new_size);
922         avl_tree_change_key(&address_space->by_addr_tree, &block->by_addr_node, &address);
923
924         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
925         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
926
927         combine_blocks_backward(new_block);
928     }
929
930     if (block->size > size)
931     {
932         memory_block_t *new_block = mem_tree_alloc();
933         new_block->flags = MEMORY_BLOCK_FREE;
934         new_block->address = (qword_t)(block->address + (size * PAGE_SIZE));
935         new_block->size = (qword_t)((dword_t)block->size - size);
936         new_block->address_space = address_space;
937         new_block->section = NULL;
938
939         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &size);
940
941         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
942         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
943
944         combine_blocks_forward(new_block);
945     }
946
947     block->flags = block_flags;
948     *virtual = (void*)((dword_t)block->address + PAGE_OFFSET((uintptr_t)physical));
949
950     lock_release(&address_space->lock);
951     return ERR_SUCCESS;
952 }
953
954 dword_t pin_memory(const void *virtual, void **pinned, uintptr_t size, bool_t lock_contents)
955 {
956     uintptr_t i;
957     uintptr_t virt_addr = PAGE_ALIGN((uintptr_t)virtual);
958     void *address = (void*)PAGE_ALIGN((uintptr_t)*pinned);
959     size = PAGE_ALIGN_UP(size) >> 12;
960
961     memory_address_space_t *address_space = check_usermode(virtual, 1) ? &get_current_process()->memory_space : &kernel_address_space;
962     lock_acquire_shared(&address_space->lock);
963     lock_acquire(&mapping_space.lock);
964
965     memory_block_t *block = find_free_block(&mapping_space, address, size);
966     if (block == NULL)
967     {
968         lock_release(&address_space->lock);
969         lock_release(&mapping_space.lock);
970         return ERR_NOMEMORY;
971     }
972
973     dword_t real_address = (address != NULL) ? (dword_t)address : (dword_t)block->address;
974     dword_t new_flags = PAGE_PRESENT | PAGE_GLOBAL;
975     if (!lock_contents) new_flags |= PAGE_WRITABLE;
976
977     for (i = 0; i < size; i++)
978     {
979         void *virt_page = (void*)(virt_addr + i * PAGE_SIZE);
980         void *phys_page = get_physical_address(virt_page);
981
982         if (lock_contents)
983         {
984             memory_block_t *block = find_block_by_addr(address_space, (void*)(virt_addr + i));
985             ASSERT(block != NULL);
986             block->flags |= MEMORY_BLOCK_COPY_ON_WRITE;
987             set_page_flags(virt_page, get_page_flags(virt_page) & ~PAGE_WRITABLE);
988         }
989
990         dword_t ret = map_page(phys_page, (void*)(real_address + i), new_flags);
991         ASSERT(ret == ERR_SUCCESS);
992         reference_page(phys_page);
993     }
994
995     if ((dword_t)block->address < (dword_t)address)
996     {
997         memory_block_t *new_block = mem_tree_alloc();
998         new_block->flags = MEMORY_BLOCK_FREE;
999         new_block->address = block->address;
1000         new_block->size = (size_t)(((dword_t)address - block->address) / PAGE_SIZE);
1001         new_block->address_space = &mapping_space;
1002         new_block->section = NULL;
1003
1004         size_t new_size = block->size - new_block->size;
1005         avl_tree_change_key(&mapping_space.by_size_tree, &block->by_size_node, &new_size);
1006         avl_tree_change_key(&mapping_space.by_addr_tree, &block->by_addr_node, &address);
1007
1008         avl_tree_insert(&mapping_space.by_addr_tree, &new_block->by_addr_node);
1009         avl_tree_insert(&mapping_space.by_size_tree, &new_block->by_size_node);
1010
1011         combine_blocks_backward(new_block);
1012     }
1013
1014     if ((dword_t)block->size > size)
1015     {
1016         memory_block_t *new_block = mem_tree_alloc();
1017         new_block->flags = MEMORY_BLOCK_FREE;
1018         new_block->address = (qword_t)(block->address + (size * PAGE_SIZE));
1019         new_block->size = (qword_t)((dword_t)block->size - size);
1020         new_block->address_space = &mapping_space;
1021         new_block->section = NULL;
1022
1023         avl_tree_change_key(&mapping_space.by_size_tree, &block->by_size_node, &size);
1024
1025         avl_tree_insert(&mapping_space.by_addr_tree, &new_block->by_addr_node);
1026         avl_tree_insert(&mapping_space.by_size_tree, &new_block->by_size_node);
1027
1028         combine_blocks_forward(new_block);
1029     }
1030
1031     block->flags = MEMORY_BLOCK_ACCESSIBLE;
1032     if (!lock_contents) block->flags |= MEMORY_BLOCK_WRITABLE;
1033     *pinned = (void*)((dword_t)block->address) + PAGE_OFFSET((uintptr_t)virtual);
1034
1035     lock_release(&address_space->lock);
1036     lock_release(&mapping_space.lock);
1037     return ERR_SUCCESS;
1038 }
1039
1040 dword_t unmap_memory_in_address_space(memory_address_space_t *address_space, void *virtual)
1041 {
1042     lock_acquire(&mapping_space.lock);
1043     uintptr_t aligned_address = PAGE_ALIGN((uintptr_t)virtual);
1044
1045     avl_node_t *node = avl_tree_lookup(&mapping_space.by_addr_tree, &aligned_address);
1046     if (node == NULL)
1047     {
1048         lock_release(&mapping_space.lock);
1049         return ERR_INVALID;
1050     }
1051
1052     memory_block_t *mem_block = CONTAINER_OF(node, memory_block_t, by_addr_node);
1053     if (mem_block->flags & MEMORY_BLOCK_FREE)
1054     {
1055         lock_release(&mapping_space.lock);
1056         return ERR_INVALID;
1057     }
1058
1059     unmap_memory_internal((void*)((dword_t)mem_block->address), (dword_t)mem_block->size * PAGE_SIZE);
1060
1061     mem_block->flags = MEMORY_BLOCK_FREE;
1062     mem_block = combine_blocks_backward(mem_block);
1063     mem_block = combine_blocks_forward(mem_block);
1064
1065     lock_release(&mapping_space.lock);
1066     return ERR_SUCCESS;
1067 }
1068
1069 dword_t map_memory(void *physical, void **virtual, uintptr_t size, dword_t block_flags)
1070 {
1071     return map_memory_in_address_space(&mapping_space, physical, virtual, size, block_flags);
1072 }
1073
1074 dword_t unmap_memory(void *virtual)
1075 {
1076     return unmap_memory_in_address_space(&mapping_space, virtual);
1077 }
1078
1079 dword_t alloc_memory_in_address_space(memory_address_space_t *address_space,
1080                                       void **address,
1081                                       dword_t size,
1082                                       dword_t block_flags,
1083                                       memory_section_t *section,
1084                                       qword_t section_offset)
1085 {
1086     void *base_address = (void*)PAGE_ALIGN((uintptr_t)*address);
1087
1088     block_flags &= ~(MEMORY_BLOCK_FREE | MEMORY_BLOCK_COPY_ON_WRITE);
1089     size = PAGE_ALIGN_UP(size) >> 12;
1090     if (size == 0) return ERR_INVALID;
1091
1092     lock_acquire(&address_space->lock);
1093
1094     memory_block_t *block = find_free_block(address_space, base_address, size);
1095     if (block == NULL)
1096     {
1097         lock_release(&address_space->lock);
1098         return ERR_NOMEMORY;
1099     }
1100
1101     if (section)
1102     {
1103         reference(&section->header);
1104         block->section = section;
1105         block->section_offset = section_offset;
1106
1107         if ((section->flags & (MEMORY_SECTION_WRITABLE | MEMORY_SECTION_DIRECT_WRITE)) == MEMORY_SECTION_WRITABLE)
1108         {
1109             block_flags |= MEMORY_BLOCK_COPY_ON_WRITE;
1110         }
1111     }
1112
1113     if ((dword_t)block->address < (dword_t)base_address)
1114     {
1115         memory_block_t *new_block = mem_tree_alloc();
1116         new_block->flags = MEMORY_BLOCK_FREE;
1117         new_block->address = block->address;
1118         new_block->size = (size_t)(((dword_t)base_address - block->address) / PAGE_SIZE);
1119         new_block->address_space = address_space;
1120         new_block->section = NULL;
1121
1122         size_t new_size = block->size - new_block->size;
1123         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &new_size);
1124         avl_tree_change_key(&address_space->by_addr_tree, &block->by_addr_node, &base_address);
1125
1126         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
1127         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
1128
1129         combine_blocks_backward(new_block);
1130     }
1131
1132     if ((dword_t)block->size > size)
1133     {
1134         memory_block_t *new_block = mem_tree_alloc();
1135         new_block->flags = MEMORY_BLOCK_FREE;
1136         new_block->address = (qword_t)(block->address + (size * PAGE_SIZE));
1137         new_block->size = (qword_t)((dword_t)block->size - size);
1138         new_block->address_space = address_space;
1139         new_block->section = NULL;
1140
1141         avl_tree_change_key(&address_space->by_size_tree, &block->by_size_node, &size);
1142
1143         avl_tree_insert(&address_space->by_addr_tree, &new_block->by_addr_node);
1144         avl_tree_insert(&address_space->by_size_tree, &new_block->by_size_node);
1145
1146         combine_blocks_forward(new_block);
1147     }
1148
1149     block->flags = block_flags;
1150     *address = (void*)((dword_t)block->address);
1151     if (block_flags & MEMORY_BLOCK_EVICTABLE) list_append(&address_space->evictable_blocks, &block->evict_link);
1152
1153     lock_release(&address_space->lock);
1154     return ERR_SUCCESS;
1155 }
1156
1157 dword_t free_memory_in_address_space(memory_address_space_t *address_space, void *address)
1158 {
1159     lock_acquire(&address_space->lock);
1160     uintptr_t aligned_address = PAGE_ALIGN((uintptr_t)address);
1161
1162     avl_node_t *node = avl_tree_lookup(&address_space->by_addr_tree, &aligned_address);
1163     if (node == NULL)
1164     {
1165         lock_release(&address_space->lock);
1166         return ERR_INVALID;
1167     }
1168
1169     memory_block_t *mem_block = CONTAINER_OF(node, memory_block_t, by_addr_node);
1170     if (mem_block->flags & MEMORY_BLOCK_FREE)
1171     {
1172         lock_release(&address_space->lock);
1173         return ERR_INVALID;
1174     }
1175
1176     release_memory_block(mem_block);
1177
1178     if (mem_block->flags & MEMORY_BLOCK_EVICTABLE) list_remove(&mem_block->evict_link);
1179     mem_block->flags = MEMORY_BLOCK_FREE;
1180
1181     mem_block = combine_blocks_backward(mem_block);
1182     mem_block = combine_blocks_forward(mem_block);
1183
1184     lock_release(&address_space->lock);
1185     return ERR_SUCCESS;
1186 }
1187
1188 dword_t commit_pages(void *address, size_t size)
1189 {
1190     uintptr_t i;
1191     uintptr_t first_page = PAGE_ALIGN((uintptr_t)address);
1192     uintptr_t last_page = PAGE_ALIGN_UP(first_page + size - 1);
1193
1194     EH_TRY
1195     {
1196         for (i = first_page; i <= last_page; i += PAGE_SIZE)
1197         {
1198             volatile uintptr_t value = *(volatile uintptr_t*)i;
1199             UNUSED_PARAMETER(value);
1200         }
1201     }
1202     EH_CATCH
1203     {
1204         EH_ESCAPE(return ERR_BADPTR);
1205     }
1206     EH_DONE;
1207
1208     return ERR_SUCCESS;
1209 }
1210
1211 dword_t uncommit_pages(void *address, size_t size)
1212 {
1213     uintptr_t i;
1214     uintptr_t first_page = PAGE_ALIGN((uintptr_t)address);
1215     uintptr_t last_page = PAGE_ALIGN_UP(first_page + size - 1);
1216
1217     EH_TRY
1218     {
1219         for (i = first_page; i <= last_page; i += PAGE_SIZE)
1220         {
1221             volatile uintptr_t value = *(volatile uintptr_t*)i;
1222             UNUSED_PARAMETER(value);
1223
1224             dword_t ret = unmap_page((void*)i);
1225             if (ret != ERR_SUCCESS) return ret;
1226         }
1227     }
1228     EH_CATCH
1229     {
1230         EH_ESCAPE(return ERR_BADPTR);
1231     }
1232     EH_DONE;
1233
1234     return ERR_SUCCESS;
1235 }
1236
1237 dword_t read_physical(void *physical, void *buffer, dword_t size)
1238 {
1239     critical_t critical;
1240     dword_t ret = ERR_SUCCESS;
1241     dword_t page;
1242     dword_t first_page = PAGE_ALIGN((dword_t)physical);
1243     dword_t last_page = PAGE_ALIGN((dword_t)physical + size - 1);
1244     dword_t offset = PAGE_OFFSET((dword_t)physical);
1245
1246     enter_critical(&critical);
1247
1248     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1249     {
1250         dword_t length = ((page == last_page) ? ((dword_t)physical + size - page) : PAGE_SIZE) - offset;
1251
1252         void *mapping = map_temporary_page((void*)page, PAGE_PRESENT);
1253         if (mapping == NULL) return ERR_NOMEMORY;
1254
1255         memcpy(buffer, (void*)((dword_t)mapping + offset), length);
1256         unmap_temporary_page(mapping);
1257
1258         buffer = (void*)((dword_t)buffer + length);
1259         offset = 0;
1260     }
1261
1262     leave_critical(&critical);
1263     return ret;
1264 }
1265
1266 dword_t write_physical(void *physical, void *buffer, dword_t size)
1267 {
1268     critical_t critical;
1269     dword_t ret = ERR_SUCCESS;
1270     dword_t page;
1271     dword_t first_page = PAGE_ALIGN((dword_t)physical);
1272     dword_t last_page = PAGE_ALIGN((dword_t)physical + size - 1);
1273     dword_t offset = PAGE_OFFSET((dword_t)physical);
1274
1275     enter_critical(&critical);
1276
1277     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1278     {
1279         dword_t length = ((page == last_page) ? ((dword_t)physical + size - page) : PAGE_SIZE) - offset;
1280
1281         void *mapping = map_temporary_page((void*)page, PAGE_PRESENT | PAGE_WRITABLE);
1282         if (mapping == NULL) return ERR_NOMEMORY;
1283
1284         memcpy((void*)((dword_t)mapping + offset), buffer, length);
1285         unmap_temporary_page(mapping);
1286
1287         buffer = (void*)((dword_t)buffer + length);
1288         offset = 0;
1289     }
1290
1291     leave_critical(&critical);
1292     return ret;
1293 }
1294
1295 sysret_t syscall_alloc_memory(handle_t process, void **address, dword_t size, dword_t flags)
1296 {
1297     process_t *proc;
1298     dword_t ret = ERR_SUCCESS;
1299     void *safe_address;
1300     void **local_address = address;
1301
1302     if (get_previous_mode() == USER_MODE)
1303     {
1304         flags &= MEMORY_BLOCK_WRITABLE | MEMORY_BLOCK_ACCESSIBLE;
1305         flags |= MEMORY_BLOCK_USERMODE | MEMORY_BLOCK_EVICTABLE;
1306
1307         if (!check_usermode(address, sizeof(void*))) return ERR_BADPTR;
1308
1309         EH_TRY
1310         {
1311             safe_address = *address;
1312             local_address = &safe_address;
1313         }
1314         EH_CATCH
1315         {
1316             EH_ESCAPE(return ERR_BADPTR);
1317         }
1318         EH_DONE;
1319     }
1320
1321     if (process != INVALID_HANDLE)
1322     {
1323         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1324     }
1325     else
1326     {
1327         proc = get_current_process();
1328         reference(&proc->header);
1329     }
1330
1331     ret = alloc_memory_in_address_space(&proc->memory_space, local_address, size, flags, NULL, 0ULL);
1332
1333     if (get_previous_mode() == USER_MODE)
1334     {
1335         EH_TRY *address = safe_address;
1336         EH_DONE;
1337     }
1338
1339     dereference(&proc->header);
1340     return ret;
1341 }
1342
1343 sysret_t syscall_free_memory(handle_t process, void *address)
1344 {
1345     dword_t ret = ERR_SUCCESS;
1346     process_t *proc;
1347
1348     if (process != INVALID_HANDLE)
1349     {
1350         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1351     }
1352     else
1353     {
1354         proc = get_current_process();
1355         reference(&proc->header);
1356     }
1357
1358     ret = free_memory_in_address_space(&proc->memory_space, address);
1359
1360     dereference(&proc->header);
1361     return ret;
1362 }
1363
1364 sysret_t syscall_commit_memory(handle_t process, void *address, dword_t size)
1365 {
1366     dword_t ret = ERR_SUCCESS;
1367     process_t *proc;
1368
1369     if (get_previous_mode() == USER_MODE && !check_usermode(address, size)) return ERR_BADPTR;
1370
1371     if (process == INVALID_HANDLE)
1372     {
1373         proc = get_current_process();
1374         reference(&proc->header);
1375     }
1376     else
1377     {
1378         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1379     }
1380
1381     if (proc->terminating) return ERR_CANCELED;
1382     lock_acquire_shared(&proc->memory_space.lock);
1383
1384     process_t *prev_proc = switch_process(proc);
1385     ret = commit_pages(address, size);
1386     switch_process(prev_proc);
1387
1388     lock_release(&proc->memory_space.lock);
1389     dereference(&proc->header);
1390     return ret;
1391 }
1392
1393 sysret_t syscall_uncommit_memory(handle_t process, void *address, dword_t size)
1394 {
1395     dword_t ret = ERR_SUCCESS;
1396     process_t *proc;
1397
1398     if (get_previous_mode() == USER_MODE && !check_usermode(address, size)) return ERR_BADPTR;
1399
1400     if (process == INVALID_HANDLE)
1401     {
1402         proc = get_current_process();
1403         reference(&proc->header);
1404     }
1405     else
1406     {
1407         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1408     }
1409
1410     if (proc->terminating) return ERR_CANCELED;
1411     lock_acquire_shared(&proc->memory_space.lock);
1412
1413     process_t *prev_proc = switch_process(proc);
1414     ret = uncommit_pages(address, size);
1415     switch_process(prev_proc);
1416
1417     lock_release(&proc->memory_space.lock);
1418     dereference(&proc->header);
1419     return ret;
1420 }
1421
1422 sysret_t syscall_set_memory_flags(handle_t process, void *address, dword_t flags)
1423 {
1424     dword_t ret = ERR_SUCCESS;
1425     process_t *proc;
1426
1427     flags &= ~(MEMORY_BLOCK_FREE | MEMORY_BLOCK_COPY_ON_WRITE);
1428     if (get_previous_mode() == USER_MODE) flags |= MEMORY_BLOCK_USERMODE | MEMORY_BLOCK_EVICTABLE;
1429
1430     if (process != INVALID_HANDLE)
1431     {
1432         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1433     }
1434     else
1435     {
1436         proc = get_current_process();
1437         reference(&proc->header);
1438     }
1439
1440     process_t *prev_proc = switch_process(proc);
1441     lock_acquire(&proc->memory_space.lock);
1442
1443     memory_block_t *block = find_block_by_addr(&proc->memory_space, address);
1444     if (block == NULL)
1445     {
1446         ret = ERR_INVALID;
1447         goto cleanup;
1448     }
1449
1450     if (block->section)
1451     {
1452         if ((flags & MEMORY_BLOCK_WRITABLE) && !(block->section->flags & MEMORY_SECTION_WRITABLE))
1453         {
1454             ret = ERR_FORBIDDEN;
1455             goto cleanup;
1456         }
1457     }
1458
1459     if (block->flags & MEMORY_BLOCK_FREE)
1460     {
1461         ret = ERR_INVALID;
1462         goto cleanup;
1463     }
1464
1465     dword_t page;
1466     dword_t start_address = (dword_t)block->address;
1467     dword_t end_address = start_address + (dword_t)block->size * PAGE_SIZE;
1468     dword_t page_flags = 0;
1469
1470     if (flags & MEMORY_BLOCK_ACCESSIBLE) page_flags |= PAGE_PRESENT;
1471     if (flags & MEMORY_BLOCK_WRITABLE) page_flags |= PAGE_WRITABLE;
1472
1473     if (flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
1474     else page_flags |= PAGE_GLOBAL;
1475
1476     for (page = start_address; page < end_address; page += PAGE_SIZE)
1477     {
1478         set_page_flags((void*)page, page_flags);
1479     }
1480
1481     if (!(block->flags & MEMORY_BLOCK_EVICTABLE) && (flags & MEMORY_BLOCK_EVICTABLE))
1482     {
1483         list_append(&proc->memory_space.evictable_blocks, &block->evict_link);
1484     }
1485     else if ((block->flags & MEMORY_BLOCK_EVICTABLE) && !(flags & MEMORY_BLOCK_EVICTABLE))
1486     {
1487         list_remove(&block->evict_link);
1488     }
1489
1490     block->flags &= MEMORY_BLOCK_COPY_ON_WRITE;
1491     block->flags |= flags;
1492
1493 cleanup:
1494     lock_release(&proc->memory_space.lock);
1495     switch_process(prev_proc);
1496     dereference(&proc->header);
1497     return ret;
1498 }
1499
1500 sysret_t syscall_query_memory(handle_t process, void *address, memory_block_info_t *info)
1501 {
1502     dword_t ret = ERR_SUCCESS;
1503     process_t *proc;
1504
1505     if ((get_previous_mode() == USER_MODE) && !check_usermode(info, sizeof(memory_block_info_t)))
1506     {
1507         return ERR_BADPTR;
1508     }
1509
1510     if (process != INVALID_HANDLE)
1511     {
1512         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1513     }
1514     else
1515     {
1516         proc = get_current_process();
1517         reference(&proc->header);
1518     }
1519
1520     lock_acquire_shared(&proc->memory_space.lock);
1521
1522     memory_block_t *block = find_block_by_addr(&proc->memory_space, address);
1523     if (block == NULL)
1524     {
1525         ret = ERR_INVALID;
1526         goto cleanup;
1527     }
1528
1529     EH_TRY
1530     {
1531         info->address = block->address;
1532         info->size = block->size;
1533         info->flags = block->flags;
1534     }
1535     EH_CATCH
1536     {
1537         ret = ERR_BADPTR;
1538     }
1539     EH_DONE;
1540
1541 cleanup:
1542     lock_release(&proc->memory_space.lock);
1543     dereference(&proc->header);
1544     return ret;
1545 }
1546
1547 sysret_t syscall_read_memory(handle_t process, void *address, void *buffer, dword_t size)
1548 {
1549     dword_t ret = ERR_SUCCESS;
1550     process_t *proc;
1551     byte_t page_cache[PAGE_SIZE];
1552
1553     if (get_previous_mode() == USER_MODE && !check_usermode(buffer, size)) return ERR_BADPTR;
1554
1555     if (process == INVALID_HANDLE)
1556     {
1557         EH_TRY
1558         {
1559             memmove(buffer, address, size);
1560             return ERR_SUCCESS;
1561         }
1562         EH_CATCH
1563         {
1564             EH_ESCAPE(return ERR_FORBIDDEN);
1565         }
1566         EH_DONE;
1567     }
1568
1569     if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1570     if (proc->terminating) return ERR_CANCELED;
1571
1572     lock_acquire_shared(&proc->memory_space.lock);
1573
1574     dword_t page;
1575     dword_t first_page = PAGE_ALIGN((dword_t)address);
1576     dword_t last_page = PAGE_ALIGN((dword_t)address + size - 1);
1577     dword_t offset = PAGE_OFFSET((dword_t)address);
1578
1579     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1580     {
1581         dword_t length = ((page == last_page) ? ((dword_t)address + size - page) : PAGE_SIZE) - offset;
1582
1583         process_t *prev_proc = switch_process(proc);
1584
1585         EH_TRY memcpy(&page_cache[offset], (void*)(page + offset), length);
1586         EH_CATCH ret = ERR_FORBIDDEN;
1587         EH_DONE;
1588
1589         switch_process(prev_proc);
1590         if (ret != ERR_SUCCESS) break;
1591
1592         EH_TRY memcpy(buffer, &page_cache[offset], length);
1593         EH_CATCH ret = ERR_BADPTR;
1594         EH_DONE;
1595
1596         buffer = (void*)((dword_t)buffer + length);
1597         offset = 0;
1598         if (ret != ERR_SUCCESS) break;
1599     }
1600
1601     lock_release(&proc->memory_space.lock);
1602     dereference(&proc->header);
1603     return ret;
1604 }
1605
1606 sysret_t syscall_write_memory(handle_t process, void *address, void *buffer, dword_t size)
1607 {
1608     dword_t ret = ERR_SUCCESS;
1609     process_t *proc;
1610     byte_t page_cache[PAGE_SIZE];
1611
1612     if (get_previous_mode() == USER_MODE && !check_usermode(buffer, size)) return ERR_BADPTR;
1613
1614     if (process == INVALID_HANDLE)
1615     {
1616         EH_TRY
1617         {
1618             memmove(address, buffer, size);
1619             return ERR_SUCCESS;
1620         }
1621         EH_CATCH
1622         {
1623             EH_ESCAPE(return ERR_FORBIDDEN);
1624         }
1625         EH_DONE;
1626     }
1627
1628     if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc)) return ERR_INVALID;
1629     if (proc->terminating) return ERR_CANCELED;
1630
1631     lock_acquire(&proc->memory_space.lock);
1632
1633     dword_t page;
1634     dword_t first_page = PAGE_ALIGN((dword_t)address);
1635     dword_t last_page = PAGE_ALIGN((dword_t)address + size - 1);
1636     dword_t offset = PAGE_OFFSET((dword_t)address);
1637
1638     for (page = first_page; page <= last_page; page += PAGE_SIZE)
1639     {
1640         dword_t length = ((page == last_page) ? ((dword_t)address + size - page) : PAGE_SIZE) - offset;
1641
1642         EH_TRY memcpy(&page_cache[offset], buffer, length);
1643         EH_CATCH ret = ERR_BADPTR;
1644         EH_DONE;
1645
1646         if (ret != ERR_SUCCESS) break;
1647         process_t *prev_proc = switch_process(proc);
1648
1649         EH_TRY memcpy((void*)(page + offset), &page_cache[offset], length);
1650         EH_CATCH ret = ERR_FORBIDDEN;
1651         EH_DONE;
1652
1653         switch_process(prev_proc);
1654
1655         buffer = (void*)((dword_t)buffer + length);
1656         offset = 0;
1657         if (ret != ERR_SUCCESS) break;
1658     }
1659
1660     lock_release(&proc->memory_space.lock);
1661     dereference(&proc->header);
1662     return ret;
1663 }
1664
1665 void *alloc_pool(void *address, dword_t size, dword_t block_flags)
1666 {
1667     size = PAGE_ALIGN_UP(size);
1668     void *result = address;
1669
1670     if (alloc_memory_in_address_space(&kernel_address_space,
1671                                       &result,
1672                                       size,
1673                                       block_flags,
1674                                       NULL,
1675                                       0ULL) == ERR_SUCCESS)
1676     {
1677         return result;
1678     }
1679     else
1680     {
1681         return NULL;
1682     }
1683 }
1684
1685 void free_pool(void *address)
1686 {
1687     free_memory_in_address_space(&kernel_address_space, address);
1688 }
1689
1690 sysret_t syscall_create_memory_section(const char *name, handle_t file, size_t max_size, dword_t flags, handle_t *handle)
1691 {
1692     dword_t ret = ERR_SUCCESS;
1693     handle_t safe_handle;
1694     char *safe_name = NULL;
1695
1696     flags &= MEMORY_SECTION_WRITABLE | MEMORY_SECTION_DIRECT_WRITE;
1697     if (flags & MEMORY_SECTION_DIRECT_WRITE) flags |= MEMORY_SECTION_WRITABLE;
1698
1699     if (get_previous_mode() == USER_MODE)
1700     {
1701         dword_t name_length = 0;
1702
1703         EH_TRY name_length = strlen(name);
1704         EH_CATCH EH_ESCAPE(return ERR_BADPTR);
1705         EH_DONE;
1706
1707         if (!check_usermode(name, name_length + 1)) return ERR_BADPTR;
1708         if (!check_usermode(handle, sizeof(handle_t))) return ERR_BADPTR;
1709
1710         safe_name = copy_user_string(name);
1711         if (safe_name == NULL) return ERR_BADPTR;
1712     }
1713     else
1714     {
1715         safe_name = (char*)name;
1716     }
1717
1718     memory_section_t *section = (memory_section_t*)malloc(sizeof(memory_section_t));
1719     if (section == NULL)
1720     {
1721         ret = ERR_NOMEMORY;
1722         goto cleanup;
1723     }
1724
1725     file_instance_t *file_instance = NULL;
1726     if (file != INVALID_HANDLE)
1727     {
1728         if (!reference_by_handle(file, OBJECT_FILE_INSTANCE, (object_t**)&file_instance))
1729         {
1730             ret = ERR_INVALID;
1731             goto cleanup;
1732         }
1733     }
1734
1735     list_init(&section->page_list);
1736     section->flags = flags;
1737     section->size = max_size;
1738     section->file = file != INVALID_HANDLE ? file_instance : NULL;
1739
1740     init_object(&section->header, safe_name, OBJECT_MEMORY);
1741     ret = create_object(&section->header);
1742     if (ret != ERR_SUCCESS)
1743     {
1744         if (file_instance) dereference(&file_instance->header);
1745         if (section->header.name) free(section->header.name);
1746         free(section);
1747         section = NULL;
1748         goto cleanup;
1749     }
1750
1751     ret = open_object(&section->header, 0, &safe_handle);
1752     if (ret == ERR_SUCCESS)
1753     {
1754         EH_TRY
1755         {
1756             *handle = safe_handle;
1757         }
1758         EH_CATCH
1759         {
1760             syscall_close_object(safe_handle);
1761             ret = ERR_BADPTR;
1762         }
1763         EH_DONE;
1764     }
1765
1766 cleanup:
1767     if (section) dereference(&section->header);
1768     if (get_previous_mode() == USER_MODE) free(safe_name);
1769
1770     return ret;
1771 }
1772
1773 sysret_t syscall_open_memory_section(const char *name, handle_t *handle)
1774 {
1775     handle_t safe_handle;
1776     char *safe_name = NULL;
1777
1778     if (get_previous_mode() == USER_MODE)
1779     {
1780         dword_t name_length = 0;
1781
1782         EH_TRY name_length = strlen(name);
1783         EH_CATCH EH_ESCAPE(return ERR_BADPTR);
1784         EH_DONE;
1785
1786         if (!check_usermode(name, name_length + 1)) return ERR_BADPTR;
1787         if (!check_usermode(handle, sizeof(handle_t))) return ERR_BADPTR;
1788
1789         safe_name = copy_user_string(name);
1790         if (safe_name == NULL) return ERR_NOMEMORY;
1791     }
1792     else safe_name = (char*)name;
1793
1794     dword_t ret = open_object_by_name(safe_name, OBJECT_MEMORY, 0, &safe_handle);
1795
1796     EH_TRY
1797     {
1798         *handle = safe_handle;
1799     }
1800     EH_CATCH
1801     {
1802         syscall_close_object(safe_handle);
1803         ret = ERR_BADPTR;
1804     }
1805     EH_DONE;
1806
1807     if (get_previous_mode() == USER_MODE) free(safe_name);
1808     return ret;
1809 }
1810
1811 sysret_t syscall_map_memory_section(handle_t process, handle_t section, void **address, qword_t offset, size_t size, dword_t flags)
1812 {
1813     dword_t ret = ERR_SUCCESS;
1814     process_t *proc = NULL;
1815     memory_section_t *mem_sec = NULL;
1816     void *safe_address;
1817
1818     if (PAGE_OFFSET(offset) != 0) return ERR_INVALID;
1819
1820     if (process != INVALID_HANDLE)
1821     {
1822         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc))
1823         {
1824             ret = ERR_INVALID;
1825             goto cleanup;
1826         }
1827     }
1828     else
1829     {
1830         proc = get_current_process();
1831         reference(&proc->header);
1832     }
1833
1834     if (!reference_by_handle(section, OBJECT_MEMORY, (object_t**)&mem_sec))
1835     {
1836         ret = ERR_INVALID;
1837         goto cleanup;
1838     }
1839
1840     if (get_previous_mode() == USER_MODE)
1841     {
1842         if (!check_usermode(address, sizeof(void*)))
1843         {
1844             ret = ERR_BADPTR;
1845             goto cleanup;
1846         }
1847
1848         EH_TRY safe_address = *address;
1849         EH_CATCH ret = ERR_BADPTR;
1850         EH_DONE;
1851
1852         if (ret != ERR_SUCCESS) goto cleanup;
1853     }
1854     else
1855     {
1856         safe_address = *address;
1857     }
1858
1859     if ((flags & MEMORY_BLOCK_WRITABLE) && !(mem_sec->flags & MEMORY_SECTION_WRITABLE))
1860     {
1861         ret = ERR_FORBIDDEN;
1862         goto cleanup;
1863     }
1864
1865     ret = alloc_memory_in_address_space(&proc->memory_space, &safe_address, size, flags, mem_sec, offset);
1866     if (ret != ERR_SUCCESS) goto cleanup;
1867
1868     EH_TRY *address = safe_address;
1869     EH_DONE;
1870
1871 cleanup:
1872     if (proc) dereference(&proc->header);
1873     if (mem_sec) dereference(&mem_sec->header);
1874     return ret;
1875 }
1876
1877 sysret_t syscall_flush_memory_section(handle_t process, void *address)
1878 {
1879     dword_t ret = ERR_SUCCESS;
1880     process_t *proc = NULL;
1881
1882     if (process != INVALID_HANDLE)
1883     {
1884         if (!reference_by_handle(process, OBJECT_PROCESS, (object_t**)&proc))
1885         {
1886             ret = ERR_INVALID;
1887             goto cleanup;
1888         }
1889     }
1890     else
1891     {
1892         proc = get_current_process();
1893         reference(&proc->header);
1894     }
1895
1896     lock_acquire_shared(&proc->memory_space.lock);
1897
1898     memory_block_t *block = find_block_by_addr(&proc->memory_space, address);
1899     if (block == NULL || block->section == NULL)
1900     {
1901         ret = ERR_INVALID;
1902         goto cleanup;
1903     }
1904
1905     if (block->section->file == NULL) goto cleanup;
1906
1907     list_entry_t *ptr;
1908
1909     for (ptr = block->section->page_list.next; ptr != &block->section->page_list; ptr = ptr->next)
1910     {
1911         dword_t bytes_written;
1912         byte_t buffer[PAGE_SIZE];
1913         shared_page_t *shared = CONTAINER_OF(ptr, shared_page_t, link);
1914
1915         ret = read_physical(shared->physical, buffer, PAGE_SIZE);
1916         if (ret != ERR_SUCCESS) continue;
1917
1918         file_instance_t *file = block->section->file;
1919         lock_acquire(&file->global->volume->lock);
1920         ret = file->global->volume->driver->write_file(file, buffer, shared->offset, PAGE_SIZE, &bytes_written);
1921         lock_release(&file->global->volume->lock);
1922         if (ret != ERR_SUCCESS) break;
1923     }
1924
1925 cleanup:
1926     lock_release(&proc->memory_space.lock);
1927     dereference(&proc->header);
1928     return ret;
1929 }
1930
1931 sysret_t syscall_add_page_file(const char *path, dword_t max_entries)
1932 {
1933     dword_t ret;
1934     char *safe_path = NULL;
1935     if (max_entries == INVALID_STORE_NUMBER) max_entries--;
1936
1937     if (get_previous_mode() == USER_MODE)
1938     {
1939         if (!check_privileges(PRIVILEGE_SET_PAGE_FILE)) return ERR_FORBIDDEN;
1940
1941         if (path)
1942         {
1943             dword_t path_length = 0;
1944
1945             EH_TRY path_length = strlen(path);
1946             EH_CATCH EH_ESCAPE(return ERR_BADPTR);
1947             EH_DONE;
1948
1949             if (!check_usermode(path, path_length + 1)) return ERR_BADPTR;
1950
1951             safe_path = copy_user_string(path);
1952             if (!safe_path) return ERR_NOMEMORY;
1953         }
1954     }
1955     else safe_path = (char*)path;
1956
1957     page_store_t *store = (page_store_t*)malloc(sizeof(page_store_t));
1958     if (store == NULL)
1959     {
1960         ret = ERR_NOMEMORY;
1961         goto cleanup;
1962     }
1963
1964     store->bitmap = malloc((max_entries + 7) / 8);
1965     if (store->bitmap == NULL)
1966     {
1967         free(store);
1968         ret = ERR_NOMEMORY;
1969         goto cleanup;
1970     }
1971
1972     memset(store->bitmap, 0, (max_entries + 7) / 8);
1973     store->num_entries = 0;
1974     store->max_entries = max_entries;
1975     list_init(&store->entry_list);
1976
1977     ret = syscall(SYSCALL_OPEN_FILE,
1978                   safe_path,
1979                   &store->file_handle,
1980                   FILE_MODE_READ
1981                   | FILE_MODE_WRITE
1982                   | FILE_MODE_NO_CACHE
1983                   | FILE_MODE_DELETE_ON_CLOSE
1984                   | FILE_MODE_CREATE
1985                   | FILE_MODE_TRUNCATE,
1986                   0);
1987     if (ret != ERR_SUCCESS)
1988     {
1989         free(store->bitmap);
1990         free(store);
1991         goto cleanup;
1992     }
1993
1994     lock_acquire(&page_store_lock);
1995     list_append(&page_stores, &store->link);
1996     lock_release(&page_store_lock);
1997
1998 cleanup:
1999     if (get_previous_mode() == USER_MODE) free(safe_path);
2000     return ret;
2001 }
2002
2003 sysret_t syscall_remove_page_file(const char *path)
2004 {
2005     dword_t ret = ERR_SUCCESS;
2006     char *safe_path = NULL;
2007
2008     if (get_previous_mode() == USER_MODE)
2009     {
2010         if (!check_privileges(PRIVILEGE_SET_PAGE_FILE)) return ERR_FORBIDDEN;
2011
2012         if (path)
2013         {
2014             dword_t path_length = 0;
2015
2016             EH_TRY path_length = strlen(path);
2017             EH_CATCH EH_ESCAPE(return ERR_BADPTR);
2018             EH_DONE;
2019
2020             if (!check_usermode(path, path_length + 1)) return ERR_BADPTR;
2021
2022             safe_path = copy_user_string(path);
2023             if (!safe_path) return ERR_NOMEMORY;
2024         }
2025     }
2026     else safe_path = (char*)path;
2027
2028     list_entry_t *ptr;
2029     page_store_t *store;
2030
2031     lock_acquire(&page_store_lock);
2032
2033     for (ptr = page_stores.next; ptr != &page_stores; ptr = ptr->next)
2034     {
2035         store = CONTAINER_OF(ptr, page_store_t, link);
2036
2037         char *name_buffer = NULL;
2038         size_t name_buffer_size = 256;
2039
2040         while (TRUE)
2041         {
2042             char *name_buffer = malloc(name_buffer_size);
2043             if (!name_buffer) break;
2044
2045             ret = syscall(SYSCALL_QUERY_FILE, store->file_handle, name_buffer, name_buffer_size);
2046             if (ret != ERR_SUCCESS) free(name_buffer);
2047             if (ret != ERR_SMALLBUF) break;
2048
2049             name_buffer_size *= 2;
2050         }
2051
2052         if (ret == ERR_SUCCESS)
2053         {
2054             bool_t found = strcmp(name_buffer, safe_path) == 0;
2055             if (name_buffer) free(name_buffer);
2056             if (found) break;
2057         }
2058     }
2059
2060     if (ptr == &page_stores)
2061     {
2062         ret = ERR_NOTFOUND;
2063         lock_release(&page_store_lock);
2064         goto cleanup;
2065     }
2066
2067     list_remove(&store->link);
2068     lock_release(&page_store_lock);
2069
2070     for (ptr = store->entry_list.next; ptr != &store->entry_list; ptr = ptr->next)
2071     {
2072         process_t *old_process;
2073         byte_t buffer[PAGE_SIZE];
2074         dword_t bytes_read;
2075         dword_t page_flags = 0;
2076         page_store_entry_t *entry = CONTAINER_OF(ptr, page_store_entry_t, link);
2077
2078         ret = syscall_read_file(store->file_handle, buffer, (qword_t)entry->number * (qword_t)PAGE_SIZE, PAGE_SIZE, &bytes_read);
2079         if (ret != ERR_SUCCESS) break;
2080
2081         lock_acquire(&entry->address_space->lock);
2082         memory_block_t *block = find_block_by_addr(entry->address_space, entry->address);
2083
2084         if (block->flags & MEMORY_BLOCK_ACCESSIBLE) page_flags |= PAGE_PRESENT;
2085         if ((block->flags & (MEMORY_BLOCK_WRITABLE | MEMORY_BLOCK_COPY_ON_WRITE))
2086             == MEMORY_BLOCK_WRITABLE)
2087         {
2088             page_flags |= PAGE_WRITABLE;
2089         }
2090
2091         if (block->flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
2092         else page_flags |= PAGE_GLOBAL;
2093
2094         if (entry->address_space != &kernel_address_space)
2095         {
2096             old_process = switch_process(CONTAINER_OF(entry->address_space, process_t, memory_space));
2097         }
2098
2099         ret = alloc_page(entry->address, page_flags);
2100         if (ret != ERR_SUCCESS) goto loop_cleanup;
2101
2102         list_entry_t *p;
2103         for (p = store->entry_list.next; p != &store->entry_list; p = ptr->next)
2104         {
2105             page_store_entry_t *other_entry = CONTAINER_OF(ptr, page_store_entry_t, link);
2106
2107             if (entry != other_entry && other_entry->number == entry->number)
2108             {
2109                 list_remove(&other_entry->link);
2110                 list_append(&transition_pages, &other_entry->link);
2111
2112                 other_entry->physical = get_physical_address(entry->address);
2113                 other_entry->number = INVALID_STORE_NUMBER;
2114             }
2115         }
2116
2117         clear_bit(store->bitmap, entry->number);
2118         list_remove(&entry->link);
2119
2120         memcpy(entry->address, buffer, PAGE_SIZE);
2121         free(entry);
2122
2123 loop_cleanup:
2124         if (entry->address_space != &kernel_address_space) switch_process(old_process);
2125         lock_release(&entry->address_space->lock);
2126     }
2127
2128     free(store);
2129
2130 cleanup:
2131     if (ret != ERR_SUCCESS)
2132     {
2133         lock_acquire(&page_store_lock);
2134         list_append(&page_stores, &store->link);
2135         lock_release(&page_store_lock);
2136     }
2137
2138     if (get_previous_mode() == USER_MODE) free(safe_path);
2139     return ret;
2140 }
2141
2142 static int compare_address(const void *key1, const void *key2)
2143 {
2144     const uintptr_t first = *(const uintptr_t*)key1;
2145     const uintptr_t second = *(const uintptr_t*)key2;
2146
2147     if (first < second) return -1;
2148     else if (first == second) return 0;
2149     else return 1;
2150 }
2151
2152 static int compare_size(const void *key1, const void *key2)
2153 {
2154     const size_t first = *(const size_t*)key1;
2155     const size_t second = *(const size_t*)key2;
2156
2157     if (first < second) return -1;
2158     else if (first == second) return 0;
2159     else return 1;
2160 }
2161
2162 dword_t create_address_space(void *base_address, dword_t page_count, memory_address_space_t *mem_space)
2163 {
2164     dword_t ret = ERR_NOMEMORY;
2165
2166     mem_space->pool_address = base_address;
2167     mem_space->pool_size = page_count;
2168     AVL_TREE_INIT(&mem_space->by_addr_tree, memory_block_t, by_addr_node, address, compare_address);
2169     AVL_TREE_INIT(&mem_space->by_size_tree, memory_block_t, by_size_node, size, compare_size);
2170     lock_init(&mem_space->lock);
2171     list_init(&mem_space->evictable_blocks);
2172     mem_space->evict_blk_ptr = NULL;
2173     mem_space->evict_page_num = 0;
2174     mem_space->stats.used_virtual = 0;
2175     mem_space->stats.committed = 0;
2176     mem_space->stats.evicted = 0;
2177     mem_space->stats.shared = 0;
2178
2179     if (get_page_directory() != INVALID_PAGE)
2180     {
2181         mem_space->page_directory = create_page_directory();
2182         if (mem_space->page_directory == NULL) return ret;
2183     }
2184     else
2185     {
2186         dword_t *boot_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
2187         mem_space->page_directory = (void*)PAGE_ALIGN(boot_directory[PAGEDIR_SELF_ENTRY]);
2188     }
2189
2190     memory_block_t *initial = mem_tree_alloc();
2191     if (initial != NULL)
2192     {
2193         initial->address = (uintptr_t)base_address;
2194         initial->size = page_count;
2195         initial->flags = MEMORY_BLOCK_FREE;
2196         initial->address_space = mem_space;
2197         initial->section = NULL;
2198
2199         avl_tree_insert(&mem_space->by_addr_tree, &initial->by_addr_node);
2200         avl_tree_insert(&mem_space->by_size_tree, &initial->by_size_node);
2201         ret = ERR_SUCCESS;
2202     }
2203
2204     if (mem_space != &kernel_address_space)
2205     {
2206         list_append(&user_address_spaces, &mem_space->link);
2207     }
2208
2209     return ret;
2210 }
2211
2212 dword_t clone_address_space(memory_address_space_t *original, memory_address_space_t *clone)
2213 {
2214     dword_t i;
2215     dword_t ret = ERR_SUCCESS;
2216
2217     lock_acquire_shared(&original->lock);
2218
2219     clone->pool_address = original->pool_address;
2220     clone->pool_size = original->pool_size;
2221     AVL_TREE_INIT(&clone->by_addr_tree, memory_block_t, by_addr_node, address, NULL);
2222     AVL_TREE_INIT(&clone->by_size_tree, memory_block_t, by_size_node, size, NULL);
2223     lock_init(&clone->lock);
2224     list_init(&clone->evictable_blocks);
2225     clone->evict_blk_ptr = NULL;
2226     clone->evict_page_num = 0;
2227     clone->stats.used_virtual = original->stats.used_virtual;
2228     clone->stats.committed = original->stats.committed;
2229     clone->stats.evicted = original->stats.evicted;
2230     clone->stats.shared = original->stats.committed;
2231
2232     if (original->by_addr_tree.root != NULL)
2233     {
2234         memory_block_t *root_block = CONTAINER_OF(original->by_addr_tree.root, memory_block_t, by_addr_node);
2235         if (!clone_blocks_recursive(clone, root_block))
2236         {
2237             ret = ERR_NOMEMORY;
2238             goto cleanup;
2239         }
2240     }
2241
2242     if (!(clone->page_directory = create_page_directory()))
2243     {
2244         ret = ERR_NOMEMORY;
2245         goto cleanup;
2246     }
2247
2248     dword_t *clone_dir = map_temporary_page(clone->page_directory, PAGE_PRESENT | PAGE_WRITABLE);
2249     bool_t this_directory = original->page_directory == get_page_directory();
2250
2251     dword_t *original_dir;
2252     if (this_directory) original_dir = (dword_t*)PAGE_DIRECTORY_ADDR;
2253     else original_dir = map_temporary_page(original->page_directory, PAGE_PRESENT | PAGE_WRITABLE);
2254
2255     for (i = USER_PAGE_START; i <= USER_PAGE_END; i++)
2256     {
2257         reference_page((void*)PAGE_ALIGN(original_dir[i]));
2258         original_dir[i] &= ~PAGE_WRITABLE;
2259         clone_dir[i] = original_dir[i];
2260         if (this_directory) cpu_invalidate_tlb((void*)(i << 12));
2261     }
2262
2263     if (!this_directory) unmap_temporary_page(original_dir);
2264     unmap_temporary_page(clone_dir);
2265     list_append(&user_address_spaces, &clone->link);
2266
2267 cleanup:
2268     lock_release(&original->lock);
2269     return ret;
2270 }
2271
2272 void bump_address_space(memory_address_space_t *mem_space)
2273 {
2274     list_remove(&mem_space->link);
2275     list_append(&user_address_spaces, &mem_space->link);
2276 }
2277
2278 void delete_address_space(memory_address_space_t *mem_space)
2279 {
2280     ASSERT(get_page_directory() != mem_space->page_directory);
2281     lock_acquire(&mem_space->lock);
2282
2283     if (mem_space->by_addr_tree.root)
2284     {
2285         memory_block_t *root = CONTAINER_OF(mem_space->by_addr_tree.root, memory_block_t, by_addr_node);
2286         free_blocks_recursive(root);
2287         mem_space->by_addr_tree.root = mem_space->by_size_tree.root = NULL;
2288     }
2289
2290     free_physical_page(mem_space->page_directory);
2291     mem_space->page_directory = NULL;
2292
2293     lock_release(&mem_space->lock);
2294 }
2295
2296 static bool_t find_evicted_page(memory_block_t *block, void *address, page_store_t **store, page_store_entry_t **entry)
2297 {
2298     list_entry_t *i;
2299
2300     for (i = transition_pages.next; i != &transition_pages; i = i->next)
2301     {
2302         *entry = CONTAINER_OF(i, page_store_entry_t, link);
2303
2304         if ((*entry)->address_space == block->address_space
2305             && PAGE_ALIGN((dword_t)(*entry)->address) == PAGE_ALIGN((dword_t)address))
2306         {
2307             return TRUE;
2308         }
2309     }
2310
2311     for (i = page_stores.next; i != &page_stores; i = i->next)
2312     {
2313         list_entry_t *j;
2314         *store = CONTAINER_OF(i, page_store_t, link);
2315
2316         for (j = (*store)->entry_list.next; j != &(*store)->entry_list; j = j->next)
2317         {
2318             *entry = CONTAINER_OF(j, page_store_entry_t, link);
2319
2320             if ((*entry)->address_space == block->address_space
2321                 && PAGE_ALIGN((dword_t)(*entry)->address) == PAGE_ALIGN((dword_t)address))
2322             {
2323                 return TRUE;
2324             }
2325         }
2326     }
2327
2328     return FALSE;
2329 }
2330
2331 bool_t memory_fault_handler(void *address, registers_t *regs)
2332 {
2333     int i;
2334     page_error_t problem;
2335     dword_t aligned_address = PAGE_ALIGN((dword_t)address);
2336     dword_t pd_index = ADDR_TO_PDE((dword_t)address);
2337     dword_t pt_index = ADDR_TO_PTE((dword_t)address);
2338     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
2339     dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + (pd_index << 12));
2340     process_t *proc = get_current_process();
2341
2342     memory_address_space_t *address_space = (proc != NULL && check_usermode(address, 1))
2343                                             ? &proc->memory_space : &kernel_address_space;
2344     memory_block_t *block = find_block_by_addr(address_space, address);
2345     if (block == NULL) return FALSE;
2346
2347     if (!(regs->error_code & PAGE_ERROR_PRESENT_FLAG))
2348     {
2349         problem = PAGE_ERROR_NOTPRESENT;
2350     }
2351     else if (!(block->flags & MEMORY_BLOCK_USERMODE)
2352         && (regs->error_code & PAGE_ERROR_USERMODE_FLAG))
2353     {
2354         problem = PAGE_ERROR_UNPRIVILEGED;
2355     }
2356     else if (regs->error_code & PAGE_ERROR_WRITE_FLAG)
2357     {
2358         problem = PAGE_ERROR_READONLY;
2359     }
2360     else
2361     {
2362         KERNEL_CRASH_WITH_REGS("Unknown paging problem", regs);
2363     }
2364
2365     if ((block->flags & MEMORY_BLOCK_ACCESSIBLE) && (problem == PAGE_ERROR_NOTPRESENT))
2366     {
2367         page_store_t *store = NULL;
2368         page_store_entry_t *entry = NULL;
2369         byte_t buffer[PAGE_SIZE];
2370         dword_t bytes_read;
2371         dword_t page_flags = 0;
2372
2373         if (find_evicted_page(block, address, &store, &entry))
2374         {
2375             if (block->flags & MEMORY_BLOCK_ACCESSIBLE) page_flags |= PAGE_PRESENT;
2376             if ((block->flags & (MEMORY_BLOCK_WRITABLE | MEMORY_BLOCK_COPY_ON_WRITE))
2377                 == MEMORY_BLOCK_WRITABLE)
2378             {
2379                 page_flags |= PAGE_WRITABLE;
2380             }
2381
2382             if (block->flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
2383             else page_flags |= PAGE_GLOBAL;
2384
2385             if (entry->number != INVALID_STORE_NUMBER)
2386             {
2387                 cpu_enable_interrupts();
2388                 dword_t ret = syscall_read_file(store->file_handle, buffer, (qword_t)entry->number * (qword_t)PAGE_SIZE, PAGE_SIZE, &bytes_read);
2389                 cpu_disable_interrupts();
2390
2391                 if ((page_directory[pd_index] & PAGE_PRESENT) && (page_table[pt_index] & PAGE_PRESENT))
2392                 {
2393                     return TRUE;
2394                 }
2395
2396                 if (ret != ERR_SUCCESS) return FALSE;
2397
2398                 ret = alloc_page((void*)aligned_address, page_flags);
2399                 if (ret != ERR_SUCCESS) return FALSE;
2400
2401                 list_entry_t *ptr;
2402                 for (ptr = store->entry_list.next; ptr != &store->entry_list; ptr = ptr->next)
2403                 {
2404                     page_store_entry_t *other_entry = CONTAINER_OF(ptr, page_store_entry_t, link);
2405
2406                     if (entry != other_entry && other_entry->number == entry->number)
2407                     {
2408                         list_remove(&other_entry->link);
2409                         list_append(&transition_pages, &other_entry->link);
2410
2411                         other_entry->physical = get_physical_address((void*)aligned_address);
2412                         other_entry->number = INVALID_STORE_NUMBER;
2413                     }
2414                 }
2415
2416                 clear_bit(store->bitmap, entry->number);
2417                 list_remove(&entry->link);
2418                 free(entry);
2419
2420                 memcpy((void*)aligned_address, buffer, PAGE_SIZE);
2421                 address_space->stats.evicted -= PAGE_SIZE;
2422                 return TRUE;
2423             }
2424             else
2425             {
2426                 if (map_page(entry->physical, entry->address, page_flags) == ERR_SUCCESS)
2427                 {
2428                     list_remove(&entry->link);
2429                     free(entry);
2430                     address_space->stats.evicted -= PAGE_SIZE;
2431                     return TRUE;
2432                 }
2433             }
2434
2435             return FALSE;
2436         }
2437         else
2438         {
2439             list_entry_t *ptr;
2440             shared_page_t *page = NULL;
2441             qword_t offset = block->section_offset + (qword_t)aligned_address - (qword_t)block->address;
2442
2443             page_flags = PAGE_PRESENT;
2444             if (block->flags & MEMORY_BLOCK_WRITABLE) page_flags |= PAGE_WRITABLE;
2445
2446             if (block->flags & MEMORY_BLOCK_USERMODE) page_flags |= PAGE_USERMODE;
2447             else page_flags |= PAGE_GLOBAL;
2448
2449             if (block->section && offset < (qword_t)block->section->size)
2450             {
2451                 ASSERT(PAGE_OFFSET(offset) == 0);
2452
2453                 for (ptr = block->section->page_list.next; ptr != &block->section->page_list; ptr = ptr->next)
2454                 {
2455                     page = CONTAINER_OF(ptr, shared_page_t, link);
2456                     if (page->offset == offset) break;
2457                 }
2458
2459                 if (ptr != &block->section->page_list)
2460                 {
2461                     return (map_page(page->physical, (void*)aligned_address, page_flags) == ERR_SUCCESS);
2462                 }
2463             }
2464
2465             memset(buffer, 0, PAGE_SIZE);
2466
2467             if (block->section && block->section->file && offset < (qword_t)block->section->size)
2468             {
2469                 cpu_enable_interrupts();
2470                 file_instance_t *file = block->section->file;
2471                 lock_acquire_shared(&file->global->volume->lock);
2472                 dword_t ret = file->global->volume->driver->read_file(file, buffer, offset, PAGE_SIZE, &bytes_read);
2473                 lock_release(&file->global->volume->lock);
2474                 cpu_disable_interrupts();
2475                 if (ret != ERR_SUCCESS && ret != ERR_BEYOND) return FALSE;
2476             }
2477
2478             dword_t ret = alloc_page((void*)aligned_address, page_flags | PAGE_WRITABLE);
2479             if (ret != ERR_SUCCESS) return FALSE;
2480
2481             memcpy((void*)aligned_address, buffer, PAGE_SIZE);
2482             set_page_flags((void*)aligned_address, page_flags);
2483
2484             if (block->section && offset < (qword_t)block->section->size)
2485             {
2486                 page = (shared_page_t*)malloc(sizeof(shared_page_t));
2487                 if (page == NULL)
2488                 {
2489                     free_page((void*)aligned_address);
2490                     return FALSE;
2491                 }
2492
2493                 page->physical = get_physical_address((void*)aligned_address);
2494                 page->offset = offset;
2495
2496                 list_append(&block->section->page_list, &page->link);
2497             }
2498
2499             address_space->stats.committed += PAGE_SIZE;
2500             return TRUE;
2501         }
2502     }
2503
2504     if ((block->flags & (MEMORY_BLOCK_COPY_ON_WRITE | MEMORY_BLOCK_WRITABLE))
2505         == (MEMORY_BLOCK_COPY_ON_WRITE | MEMORY_BLOCK_WRITABLE)
2506         && (problem == PAGE_ERROR_READONLY))
2507     {
2508         if (!(page_directory[pd_index] & PAGE_WRITABLE))
2509         {
2510             void *table_phys = (void*)PAGE_ALIGN(page_directory[pd_index]);
2511
2512             if (get_page(table_phys)->ref_count > 1)
2513             {
2514                 void *table_copy = alloc_physical_page();
2515                 if (table_copy == NULL) return FALSE;
2516
2517                 dword_t *temporary = map_temporary_page(table_copy, PAGE_PRESENT | PAGE_WRITABLE);
2518                 if (temporary == NULL)
2519                 {
2520                     free_physical_page(table_copy);
2521                     return FALSE;
2522                 }
2523
2524                 for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++)
2525                 {
2526                     if (page_table[i])
2527                     {
2528                         reference_page((void*)PAGE_ALIGN(page_table[i]));
2529                         temporary[i] = page_table[i] & ~PAGE_WRITABLE;
2530                     }
2531                 }
2532
2533                 unmap_temporary_page(temporary);
2534
2535                 reference_page(table_copy);
2536                 dereference_page(table_phys);
2537
2538                 page_directory[pd_index] = PAGE_ALIGN((dword_t)table_copy)
2539                                            | PAGE_OFFSET(page_directory[pd_index])
2540                                            | PAGE_WRITABLE;
2541                 cpu_invalidate_tlb(page_table);
2542             }
2543             else
2544             {
2545                 page_directory[pd_index] |= PAGE_WRITABLE;
2546                 cpu_invalidate_tlb(page_table);
2547
2548                 for (i = 0; i < PAGE_SIZE / sizeof(dword_t); i++)
2549                 {
2550                     page_table[i] &= ~PAGE_WRITABLE;
2551                     cpu_invalidate_tlb((void*)((pd_index << 22) | (i << 12)));
2552                 }
2553             }
2554         }
2555
2556         if (!(page_table[pt_index] & PAGE_WRITABLE))
2557         {
2558             void *phys = (void*)PAGE_ALIGN(page_table[pt_index]);
2559
2560             if (get_page(phys)->ref_count > 1)
2561             {
2562                 void *page_copy = alloc_physical_page();
2563                 if (page_copy == NULL) return FALSE;
2564
2565                 write_physical(page_copy, (void*)PAGE_ALIGN((dword_t)address), PAGE_SIZE);
2566                 reference_page(page_copy);
2567                 dereference_page(phys);
2568
2569                 page_table[pt_index] = PAGE_ALIGN((dword_t)page_copy)
2570                                        | PAGE_OFFSET(page_table[pt_index])
2571                                        | PAGE_WRITABLE;
2572                 cpu_invalidate_tlb((void*)aligned_address);
2573             }
2574             else
2575             {
2576                 page_table[pt_index] |= PAGE_WRITABLE;
2577                 cpu_invalidate_tlb((void*)aligned_address);
2578             }
2579         }
2580
2581         return TRUE;
2582     }
2583
2584     return FALSE;
2585 }
2586
2587 void memory_init(multiboot_tag_mmap_t *mmap, uintptr_t lowest_physical)
2588 {
2589     dword_t i, j;
2590     dword_t *page_directory = (dword_t*)PAGE_DIRECTORY_ADDR;
2591
2592     fix_overlapping_sections(mmap);
2593
2594     log_write(LOG_NORMAL, "Memory map:\nBase\t\t\tLength\t\t\tType\n");
2595     log_write(LOG_NORMAL, "------------------------------------------------------------\n");
2596
2597     multiboot_mmap_entry_t *entry;
2598
2599     for (entry = (multiboot_mmap_entry_t*)(mmap + 1);
2600          (uintptr_t)entry < ((uintptr_t)mmap + mmap->size);
2601          entry = (multiboot_mmap_entry_t*)((uintptr_t)entry + mmap->entry_size))
2602     {
2603         log_write(LOG_NORMAL, "0x%08X%08X\t0x%08X%08X\t%s\n",
2604                   entry->base_high,
2605                   entry->base_low,
2606                   entry->length_high,
2607                   entry->length_low,
2608                   (entry->type == 1) ? "Usable" : "Not Usable");
2609
2610         if (entry->type == 1
2611             && entry->base_high == 0
2612             && entry->length_high == 0
2613             && entry->length_low < (0xFFFFFFFF - entry->base_low)
2614             && entry->length_low > 0)
2615         {
2616             dword_t start_addr = entry->base_low;
2617             if (start_addr < lowest_physical) start_addr = lowest_physical;
2618             start_addr = PAGE_ALIGN_UP(start_addr);
2619             dword_t end_addr = PAGE_ALIGN_UP(entry->base_low + entry->length_low);
2620             dword_t page = end_addr - PAGE_SIZE;
2621
2622             while (page >= start_addr)
2623             {
2624                 dword_t stack_address = (dword_t)&physical_memory_stack[num_free_pages];
2625                 dword_t pd_index = ADDR_TO_PDE(stack_address);
2626                 dword_t pt_index = ADDR_TO_PTE(stack_address);
2627                 dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + pd_index * PAGE_SIZE);
2628
2629                 if (!(page_directory[pd_index] & PAGE_PRESENT))
2630                 {
2631                     page_directory[pd_index] = start_addr | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2632                     start_addr += PAGE_SIZE;
2633                     cpu_invalidate_tlb(page_table);
2634                     memset(page_table, 0, PAGE_SIZE);
2635                     total_physical_pages++;
2636                     continue;
2637                 }
2638
2639                 if (!(page_table[pt_index] & PAGE_PRESENT))
2640                 {
2641                     page_table[pt_index] = start_addr | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2642                     start_addr += PAGE_SIZE;
2643                     cpu_invalidate_tlb((void*)stack_address);
2644                     total_physical_pages++;
2645                     continue;
2646                 }
2647
2648                 free_physical_page((void*)page);
2649                 page -= PAGE_SIZE;
2650             }
2651         }
2652     }
2653
2654     log_write(LOG_NORMAL, "------------------------------------------------------------\n");
2655     total_physical_pages += num_free_pages;
2656     pages = (page_t*)(KERNEL_POOL_START - total_physical_pages * sizeof(page_t));
2657
2658     for (i = PAGE_ALIGN((uintptr_t)pages); i < KERNEL_POOL_START; i += PAGE_SIZE)
2659     {
2660         dword_t pd_index = ADDR_TO_PDE(i);
2661         dword_t pt_index = ADDR_TO_PTE(i);
2662         dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + pd_index * PAGE_SIZE);
2663
2664         if (!(page_directory[pd_index] & PAGE_PRESENT))
2665         {
2666             page_directory[pd_index] = (uintptr_t)alloc_physical_page() | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2667             cpu_invalidate_tlb(page_table);
2668             memset(page_table, 0, PAGE_SIZE);
2669         }
2670
2671         if (!(page_table[pt_index] & PAGE_PRESENT))
2672         {
2673             page_table[pt_index] = (uintptr_t)alloc_physical_page() | PAGE_PRESENT | PAGE_WRITABLE | PAGE_GLOBAL;
2674             cpu_invalidate_tlb((void*)i);
2675         }
2676     }
2677
2678     dword_t pages_inserted = 0;
2679
2680     for (i = 0; i < num_free_pages; i++)
2681     {
2682         pages[pages_inserted].phys_addr = PAGE_ALIGN((dword_t)physical_memory_stack[i]);
2683         pages[pages_inserted].ref_count = 0;
2684         pages_inserted++;
2685     }
2686
2687     for (i = KERNEL_PAGE_START; i <= KERNEL_PAGE_END; i++)
2688     {
2689         dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + i * PAGE_SIZE);
2690         if (!(page_directory[i] & PAGE_PRESENT)) continue;
2691
2692         for (j = 0; j < PAGE_SIZE / sizeof(dword_t); j++)
2693         {
2694             if (PAGE_ALIGN(page_table[j]) < lowest_physical) continue;
2695
2696             if (page_table[j] & PAGE_PRESENT)
2697             {
2698                 pages[pages_inserted].phys_addr = PAGE_ALIGN((dword_t)page_table[j]);
2699                 pages[pages_inserted].ref_count = 0;
2700                 pages_inserted++;
2701             }
2702         }
2703     }
2704
2705     ASSERT(pages_inserted == total_physical_pages);
2706     qsort(pages, total_physical_pages, sizeof(page_t), compare_page);
2707
2708     init_semaphore(&temporary_page_semaphore, TEMPORARY_PAGES, TEMPORARY_PAGES);
2709
2710     if (create_address_space((void*)KERNEL_POOL_START,
2711                              (KERNEL_POOL_END - KERNEL_POOL_START + PAGE_SIZE - 1) / PAGE_SIZE,
2712                              &kernel_address_space) != ERR_SUCCESS)
2713     {
2714         KERNEL_CRASH("Unable to create kernel address space");
2715     }
2716
2717     if (create_address_space((void*)MAPPING_START,
2718                              (MAPPING_END - MAPPING_START + PAGE_SIZE - 1) / PAGE_SIZE,
2719                              &mapping_space) != ERR_SUCCESS)
2720     {
2721         KERNEL_CRASH("Unable to create mapping space");
2722     }
2723
2724     set_page_directory((void*)PAGE_ALIGN(page_directory[PAGEDIR_SELF_ENTRY]));
2725
2726     for (i = KERNEL_PAGE_START; i <= KERNEL_PAGE_END; i++)
2727     {
2728         dword_t *page_table = (dword_t*)(PAGE_TABLE_ADDR + i * PAGE_SIZE);
2729         if (!(page_directory[i] & PAGE_PRESENT)) continue;
2730
2731         for (j = 0; j < PAGE_SIZE / sizeof(dword_t); j++)
2732         {
2733             if (page_table[j] & PAGE_PRESENT) reference_page((void*)PAGE_ALIGN(page_table[j]));
2734         }
2735     }
2736
2737     for (i = USER_PAGE_START; i <= USER_PAGE_END; i++) page_directory[i] = 0;
2738     set_page_directory(get_page_directory());
2739 }