vm_apple_protect.c [plain text]
#include <sys/errno.h>
#include <mach/mach_types.h>
#include <mach/mach_traps.h>
#include <mach/host_priv.h>
#include <mach/kern_return.h>
#include <mach/memory_object_control.h>
#include <mach/memory_object_types.h>
#include <mach/port.h>
#include <mach/policy.h>
#include <mach/upl.h>
#include <mach/thread_act.h>
#include <mach/mach_vm.h>
#include <kern/host.h>
#include <kern/kalloc.h>
#include <kern/page_decrypt.h>
#include <kern/queue.h>
#include <kern/thread.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <default_pager/default_pager_types.h>
#include <default_pager/default_pager_object_server.h>
#include <vm/vm_map.h>
#include <vm/vm_pageout.h>
#include <vm/memory_object.h>
#include <vm/vm_pageout.h>
#include <vm/vm_protos.h>
void apple_protect_pager_reference(memory_object_t mem_obj);
void apple_protect_pager_deallocate(memory_object_t mem_obj);
kern_return_t apple_protect_pager_init(memory_object_t mem_obj,
memory_object_control_t control,
vm_size_t pg_size);
kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
memory_object_offset_t offset,
vm_size_t length,
vm_prot_t protection_required);
kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
memory_object_offset_t offset,
vm_size_t data_cnt,
memory_object_offset_t *resid_offset,
int *io_error,
boolean_t dirty,
boolean_t kernel_copy,
int upl_flags);
kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,
memory_object_offset_t offset,
vm_size_t data_cnt);
kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,
memory_object_offset_t offset,
vm_size_t size,
vm_prot_t desired_access);
kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,
memory_object_offset_t offset,
vm_size_t length,
vm_sync_t sync_flags);
kern_return_t apple_protect_pager_unmap(memory_object_t mem_obj);
const struct memory_object_pager_ops apple_protect_pager_ops = {
apple_protect_pager_reference,
apple_protect_pager_deallocate,
apple_protect_pager_init,
apple_protect_pager_terminate,
apple_protect_pager_data_request,
apple_protect_pager_data_return,
apple_protect_pager_data_initialize,
apple_protect_pager_data_unlock,
apple_protect_pager_synchronize,
apple_protect_pager_unmap,
"apple protect pager"
};
typedef struct apple_protect_pager {
memory_object_pager_ops_t pager_ops;
unsigned int pager_ikot;
queue_chain_t pager_queue;
unsigned int ref_count;
boolean_t is_ready;
boolean_t is_mapped;
memory_object_control_t pager_control;
vm_object_t backing_object;
} *apple_protect_pager_t;
#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL)
int apple_protect_pager_count = 0;
int apple_protect_pager_count_mapped = 0;
queue_head_t apple_protect_pager_queue;
decl_mutex_data(,apple_protect_pager_lock)
int apple_protect_pager_cache_limit = 10;
int apple_protect_pager_count_max = 0;
int apple_protect_pager_count_unmapped_max = 0;
int apple_protect_pager_num_trim_max = 0;
int apple_protect_pager_num_trim_total = 0;
apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object);
apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
void apple_protect_pager_dequeue(apple_protect_pager_t pager);
void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,
boolean_t locked);
void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
void apple_protect_pager_trim(void);
#if DEBUG
int apple_protect_pagerdebug = 0;
#define PAGER_ALL 0xffffffff
#define PAGER_INIT 0x00000001
#define PAGER_PAGEIN 0x00000002
#define PAGER_DEBUG(LEVEL, A) \
MACRO_BEGIN \
if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \
printf A; \
} \
MACRO_END
#else
#define PAGER_DEBUG(LEVEL, A)
#endif
void
apple_protect_pager_bootstrap(void)
{
mutex_init(&apple_protect_pager_lock, 0);
queue_init(&apple_protect_pager_queue);
}
kern_return_t
apple_protect_pager_init(
memory_object_t mem_obj,
memory_object_control_t control,
#if !DEBUG
__unused
#endif
vm_size_t pg_size)
{
apple_protect_pager_t pager;
kern_return_t kr;
memory_object_attr_info_data_t attributes;
PAGER_DEBUG(PAGER_ALL,
("apple_protect_pager_init: %p, %p, %x\n",
mem_obj, control, pg_size));
if (control == MEMORY_OBJECT_CONTROL_NULL)
return KERN_INVALID_ARGUMENT;
pager = apple_protect_pager_lookup(mem_obj);
memory_object_control_reference(control);
pager->pager_control = control;
attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
attributes.cluster_size = (1 << (PAGE_SHIFT));
attributes.may_cache_object = FALSE;
attributes.temporary = TRUE;
kr = memory_object_change_attributes(
control,
MEMORY_OBJECT_ATTRIBUTE_INFO,
(memory_object_info_t) &attributes,
MEMORY_OBJECT_ATTR_INFO_COUNT);
if (kr != KERN_SUCCESS)
panic("apple_protect_pager_init: "
"memory_object_change_attributes() failed");
return KERN_SUCCESS;
}
kern_return_t
apple_protect_pager_data_return(
__unused memory_object_t mem_obj,
__unused memory_object_offset_t offset,
__unused vm_size_t data_cnt,
__unused memory_object_offset_t *resid_offset,
__unused int *io_error,
__unused boolean_t dirty,
__unused boolean_t kernel_copy,
__unused int upl_flags)
{
panic("apple_protect_pager_data_return: should never get called");
return KERN_FAILURE;
}
kern_return_t
apple_protect_pager_data_initialize(
__unused memory_object_t mem_obj,
__unused memory_object_offset_t offset,
__unused vm_size_t data_cnt)
{
panic("apple_protect_pager_data_initialize: should never get called");
return KERN_FAILURE;
}
kern_return_t
apple_protect_pager_data_unlock(
__unused memory_object_t mem_obj,
__unused memory_object_offset_t offset,
__unused vm_size_t size,
__unused vm_prot_t desired_access)
{
return KERN_FAILURE;
}
kern_return_t
apple_protect_pager_data_request(
memory_object_t mem_obj,
memory_object_offset_t offset,
vm_size_t length,
#if !DEBUG
__unused
#endif
vm_prot_t protection_required)
{
apple_protect_pager_t pager;
memory_object_control_t mo_control;
upl_t upl = NULL;
int upl_flags;
upl_size_t upl_size;
upl_page_info_t *upl_pl;
vm_object_t src_object, dst_object;
kern_return_t kr, retval;
vm_map_offset_t src_mapping = 0, dst_mapping = 0;
vm_offset_t src_vaddr, dst_vaddr;
vm_offset_t cur_offset;
boolean_t src_map_page_by_page;
vm_map_entry_t map_entry;
PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %x, %llx, %llxx, %x\n", mem_obj, offset, length, protection_required));
pager = apple_protect_pager_lookup(mem_obj);
assert(pager->is_ready);
assert(pager->ref_count > 1);
PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %x, %llx, %llx, %x, pager %x\n", mem_obj, offset, length, protection_required, pager));
src_object = pager->backing_object;
assert(src_object != VM_OBJECT_NULL);
vm_object_reference(src_object);
src_mapping = 0;
kr = vm_map_enter(kernel_map,
&src_mapping,
length,
0,
VM_FLAGS_ANYWHERE,
src_object,
offset,
FALSE,
VM_PROT_READ,
VM_PROT_READ,
VM_INHERIT_NONE);
switch (kr) {
case KERN_SUCCESS:
kr = vm_map_wire(kernel_map,
src_mapping,
src_mapping + length,
VM_PROT_READ,
FALSE);
if (kr != KERN_SUCCESS) {
kr = vm_map_remove(kernel_map,
src_mapping,
src_mapping + length,
VM_MAP_NO_FLAGS);
assert(kr == KERN_SUCCESS);
src_mapping = 0;
src_vaddr = 0;
src_map_page_by_page = TRUE;
break;
}
src_map_page_by_page = FALSE;
src_vaddr = CAST_DOWN(vm_offset_t, src_mapping);
break;
case KERN_NO_SPACE:
src_map_page_by_page = TRUE;
vm_object_deallocate(src_object);
break;
default:
vm_object_deallocate(src_object);
retval = kr;
goto done;
}
mo_control = pager->pager_control;
upl_size = length;
upl_flags =
UPL_RET_ONLY_ABSENT |
UPL_SET_LITE |
UPL_NO_SYNC |
UPL_CLEAN_IN_PLACE |
UPL_SET_INTERNAL;
kr = memory_object_upl_request(mo_control,
offset, upl_size,
&upl, NULL, NULL, upl_flags);
if (kr != KERN_SUCCESS) {
retval = kr;
goto done;
}
dst_object = mo_control->moc_object;
assert(dst_object != VM_OBJECT_NULL);
dst_mapping = 0;
vm_object_reference(kernel_object);
kr = vm_map_find_space(kernel_map,
&dst_mapping,
PAGE_SIZE_64,
0,
0,
&map_entry);
if (kr != KERN_SUCCESS) {
vm_object_deallocate(kernel_object);
retval = kr;
goto done;
}
map_entry->object.vm_object = kernel_object;
map_entry->offset = dst_mapping - VM_MIN_KERNEL_ADDRESS;
vm_map_unlock(kernel_map);
dst_vaddr = CAST_DOWN(vm_offset_t, dst_mapping);
upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
for (cur_offset = 0; cur_offset < length; cur_offset += PAGE_SIZE) {
ppnum_t dst_pnum;
if (!upl_page_present(upl_pl, cur_offset / PAGE_SIZE)) {
continue;
}
if (src_map_page_by_page) {
vm_object_reference(src_object);
kr = vm_map_enter(kernel_map,
&src_mapping,
PAGE_SIZE_64,
0,
VM_FLAGS_ANYWHERE,
src_object,
offset + cur_offset,
FALSE,
VM_PROT_READ,
VM_PROT_READ,
VM_INHERIT_NONE);
if (kr != KERN_SUCCESS) {
vm_object_deallocate(src_object);
retval = kr;
goto done;
}
kr = vm_map_wire(kernel_map,
src_mapping,
src_mapping + PAGE_SIZE_64,
VM_PROT_READ,
FALSE);
if (kr != KERN_SUCCESS) {
retval = kr;
kr = vm_map_remove(kernel_map,
src_mapping,
src_mapping + PAGE_SIZE_64,
VM_MAP_NO_FLAGS);
assert(kr == KERN_SUCCESS);
src_mapping = 0;
src_vaddr = 0;
printf("apple_protect_pager_data_request: "
"failed to resolve page fault for src "
"object %p offset 0x%llx "
"preempt %d error 0x%x\n",
src_object, offset + cur_offset,
get_preemption_level(), retval);
goto done;
}
src_vaddr = CAST_DOWN(vm_offset_t, src_mapping);
} else {
src_vaddr = src_mapping + cur_offset;
}
dst_pnum = (addr64_t)
upl_phys_page(upl_pl, cur_offset / PAGE_SIZE);
assert(dst_pnum != 0);
pmap_enter(kernel_pmap, dst_mapping, dst_pnum,
VM_PROT_READ | VM_PROT_WRITE,
dst_object->wimg_bits & VM_WIMG_MASK,
FALSE);
dsmos_page_transform((const void *) src_vaddr,
(void *) dst_vaddr);
pmap_remove(kernel_pmap,
(addr64_t) dst_mapping,
(addr64_t) (dst_mapping + PAGE_SIZE_64));
if (src_map_page_by_page) {
kr = vm_map_remove(kernel_map,
src_mapping,
src_mapping + PAGE_SIZE_64,
VM_MAP_REMOVE_KUNWIRE);
assert(kr == KERN_SUCCESS);
src_mapping = 0;
src_vaddr = 0;
}
}
retval = KERN_SUCCESS;
done:
if (src_mapping != 0) {
kr = vm_map_remove(kernel_map,
src_mapping,
src_mapping + length,
VM_MAP_REMOVE_KUNWIRE);
assert(kr == KERN_SUCCESS);
src_mapping = 0;
src_vaddr = 0;
}
if (upl != NULL) {
upl_clear_dirty(upl, TRUE);
if (retval != KERN_SUCCESS) {
upl_abort(upl, 0);
} else {
upl_commit(upl, NULL, 0);
}
upl_deallocate(upl);
upl = NULL;
}
if (dst_mapping != 0) {
kr = vm_map_remove(kernel_map,
dst_mapping,
dst_mapping + PAGE_SIZE_64,
VM_MAP_NO_FLAGS);
assert(kr == KERN_SUCCESS);
dst_mapping = 0;
dst_vaddr = 0;
}
return retval;
}
void
apple_protect_pager_reference(
memory_object_t mem_obj)
{
apple_protect_pager_t pager;
pager = apple_protect_pager_lookup(mem_obj);
mutex_lock(&apple_protect_pager_lock);
assert(pager->ref_count > 0);
pager->ref_count++;
mutex_unlock(&apple_protect_pager_lock);
}
void
apple_protect_pager_dequeue(
apple_protect_pager_t pager)
{
assert(!pager->is_mapped);
queue_remove(&apple_protect_pager_queue,
pager,
apple_protect_pager_t,
pager_queue);
pager->pager_queue.next = NULL;
pager->pager_queue.prev = NULL;
apple_protect_pager_count--;
}
void
apple_protect_pager_terminate_internal(
apple_protect_pager_t pager)
{
assert(pager->is_ready);
assert(!pager->is_mapped);
if (pager->backing_object != VM_OBJECT_NULL) {
vm_object_deallocate(pager->backing_object);
pager->backing_object = VM_OBJECT_NULL;
}
memory_object_destroy(pager->pager_control, 0);
}
void
apple_protect_pager_deallocate_internal(
apple_protect_pager_t pager,
boolean_t locked)
{
boolean_t needs_trimming;
int count_unmapped;
if (! locked) {
mutex_lock(&apple_protect_pager_lock);
}
count_unmapped = (apple_protect_pager_count -
apple_protect_pager_count_mapped);
if (count_unmapped > apple_protect_pager_cache_limit) {
needs_trimming = TRUE;
} else {
needs_trimming = FALSE;
}
pager->ref_count--;
if (pager->ref_count == 1) {
apple_protect_pager_dequeue(pager);
mutex_unlock(&apple_protect_pager_lock);
apple_protect_pager_terminate_internal(pager);
} else if (pager->ref_count == 0) {
mutex_unlock(&apple_protect_pager_lock);
if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
memory_object_control_deallocate(pager->pager_control);
pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
}
kfree(pager, sizeof (*pager));
pager = APPLE_PROTECT_PAGER_NULL;
} else {
mutex_unlock(&apple_protect_pager_lock);
}
if (needs_trimming) {
apple_protect_pager_trim();
}
}
void
apple_protect_pager_deallocate(
memory_object_t mem_obj)
{
apple_protect_pager_t pager;
PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %x\n", mem_obj));
pager = apple_protect_pager_lookup(mem_obj);
apple_protect_pager_deallocate_internal(pager, FALSE);
}
kern_return_t
apple_protect_pager_terminate(
#if !DEBUG
__unused
#endif
memory_object_t mem_obj)
{
PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %x\n", mem_obj));
return KERN_SUCCESS;
}
kern_return_t
apple_protect_pager_synchronize(
memory_object_t mem_obj,
memory_object_offset_t offset,
vm_size_t length,
__unused vm_sync_t sync_flags)
{
apple_protect_pager_t pager;
PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %x\n", mem_obj));
pager = apple_protect_pager_lookup(mem_obj);
memory_object_synchronize_completed(pager->pager_control,
offset, length);
return KERN_SUCCESS;
}
void
apple_protect_pager_map(
memory_object_t mem_obj)
{
apple_protect_pager_t pager;
PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %x\n", mem_obj));
pager = apple_protect_pager_lookup(mem_obj);
mutex_lock(&apple_protect_pager_lock);
assert(pager->is_ready);
assert(pager->ref_count > 0);
if (pager->is_mapped == FALSE) {
pager->is_mapped = TRUE;
pager->ref_count++;
apple_protect_pager_count_mapped++;
}
mutex_unlock(&apple_protect_pager_lock);
}
kern_return_t
apple_protect_pager_unmap(
memory_object_t mem_obj)
{
apple_protect_pager_t pager;
int count_unmapped;
PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_unmap: %x\n", mem_obj));
pager = apple_protect_pager_lookup(mem_obj);
mutex_lock(&apple_protect_pager_lock);
if (pager->is_mapped) {
apple_protect_pager_count_mapped--;
count_unmapped = (apple_protect_pager_count -
apple_protect_pager_count_mapped);
if (count_unmapped > apple_protect_pager_count_unmapped_max) {
apple_protect_pager_count_unmapped_max = count_unmapped;
}
pager->is_mapped = FALSE;
apple_protect_pager_deallocate_internal(pager, TRUE);
} else {
mutex_unlock(&apple_protect_pager_lock);
}
return KERN_SUCCESS;
}
apple_protect_pager_t
apple_protect_pager_lookup(
memory_object_t mem_obj)
{
apple_protect_pager_t pager;
pager = (apple_protect_pager_t) mem_obj;
assert(pager->pager_ops == &apple_protect_pager_ops);
assert(pager->ref_count > 0);
return pager;
}
apple_protect_pager_t
apple_protect_pager_create(
vm_object_t backing_object)
{
apple_protect_pager_t pager, pager2;
memory_object_control_t control;
kern_return_t kr;
pager = (apple_protect_pager_t) kalloc(sizeof (*pager));
if (pager == APPLE_PROTECT_PAGER_NULL) {
return APPLE_PROTECT_PAGER_NULL;
}
pager->pager_ops = &apple_protect_pager_ops;
pager->pager_ikot = IKOT_MEMORY_OBJECT;
pager->is_ready = FALSE;
pager->ref_count = 2;
pager->is_mapped = FALSE;
pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
pager->backing_object = backing_object;
vm_object_reference(backing_object);
mutex_lock(&apple_protect_pager_lock);
queue_iterate(&apple_protect_pager_queue,
pager2,
apple_protect_pager_t,
pager_queue) {
if (pager2->backing_object == backing_object) {
break;
}
}
if (! queue_end(&apple_protect_pager_queue,
(queue_entry_t) pager2)) {
pager2->ref_count++;
mutex_unlock(&apple_protect_pager_lock);
vm_object_deallocate(pager->backing_object);
pager->backing_object = VM_OBJECT_NULL;
kfree(pager, sizeof (*pager));
pager = pager2;
return pager;
}
queue_enter_first(&apple_protect_pager_queue,
pager,
apple_protect_pager_t,
pager_queue);
apple_protect_pager_count++;
if (apple_protect_pager_count > apple_protect_pager_count_max) {
apple_protect_pager_count_max = apple_protect_pager_count;
}
mutex_unlock(&apple_protect_pager_lock);
kr = memory_object_create_named((memory_object_t) pager,
0,
&control);
assert(kr == KERN_SUCCESS);
mutex_lock(&apple_protect_pager_lock);
pager->is_ready = TRUE;
mutex_unlock(&apple_protect_pager_lock);
thread_wakeup(&pager->is_ready);
return pager;
}
memory_object_t
apple_protect_pager_setup(
vm_object_t backing_object)
{
apple_protect_pager_t pager;
mutex_lock(&apple_protect_pager_lock);
queue_iterate(&apple_protect_pager_queue,
pager,
apple_protect_pager_t,
pager_queue) {
if (pager->backing_object == backing_object) {
break;
}
}
if (queue_end(&apple_protect_pager_queue,
(queue_entry_t) pager)) {
pager = APPLE_PROTECT_PAGER_NULL;
} else {
pager->ref_count++;
}
mutex_unlock(&apple_protect_pager_lock);
if (pager == APPLE_PROTECT_PAGER_NULL) {
pager = apple_protect_pager_create(backing_object);
if (pager == APPLE_PROTECT_PAGER_NULL) {
return MEMORY_OBJECT_NULL;
}
}
mutex_lock(&apple_protect_pager_lock);
while (!pager->is_ready) {
thread_sleep_mutex(&pager->is_ready,
&apple_protect_pager_lock,
THREAD_UNINT);
}
mutex_unlock(&apple_protect_pager_lock);
return (memory_object_t) pager;
}
void
apple_protect_pager_trim(void)
{
apple_protect_pager_t pager, prev_pager;
queue_head_t trim_queue;
int num_trim;
int count_unmapped;
mutex_lock(&apple_protect_pager_lock);
queue_init(&trim_queue);
num_trim = 0;
for (pager = (apple_protect_pager_t)
queue_last(&apple_protect_pager_queue);
!queue_end(&apple_protect_pager_queue,
(queue_entry_t) pager);
pager = prev_pager) {
prev_pager = (apple_protect_pager_t)
queue_prev(&pager->pager_queue);
if (pager->ref_count == 2 &&
pager->is_ready &&
!pager->is_mapped) {
num_trim++;
apple_protect_pager_dequeue(pager);
queue_enter_first(&trim_queue,
pager,
apple_protect_pager_t,
pager_queue);
count_unmapped = (apple_protect_pager_count -
apple_protect_pager_count_mapped);
if (count_unmapped <= apple_protect_pager_cache_limit) {
break;
}
}
}
if (num_trim > apple_protect_pager_num_trim_max) {
apple_protect_pager_num_trim_max = num_trim;
}
apple_protect_pager_num_trim_total += num_trim;
mutex_unlock(&apple_protect_pager_lock);
while (!queue_empty(&trim_queue)) {
queue_remove_first(&trim_queue,
pager,
apple_protect_pager_t,
pager_queue);
pager->pager_queue.next = NULL;
pager->pager_queue.prev = NULL;
assert(pager->ref_count == 2);
pager->ref_count--;
apple_protect_pager_terminate_internal(pager);
}
}