diff options
author | Philippe Langlais <philippe.langlais@stericsson.com> | 2012-06-04 19:46:22 +0800 |
---|---|---|
committer | Philippe Langlais <philippe.langlais@stericsson.com> | 2012-06-04 19:46:22 +0800 |
commit | 1c4a401e4961ffda3c432504e69d12316cfdbdf3 (patch) | |
tree | 1fa2157f1d2712a7f838eadd236e35681957c323 /drivers/gpu/mali/mali400ko/driver/src/devicedrv/mali/common/mali_kernel_mem_mmu.c | |
parent | c65e9ef43160d318d715d5a1c8752864aace85f0 (diff) | |
parent | fcddfa7f1d6cbdd5b841665b2337422627394cff (diff) |
Merge topic branch 'mali-android' into integration-android-ux500
Diffstat (limited to 'drivers/gpu/mali/mali400ko/driver/src/devicedrv/mali/common/mali_kernel_mem_mmu.c')
-rw-r--r-- | drivers/gpu/mali/mali400ko/driver/src/devicedrv/mali/common/mali_kernel_mem_mmu.c | 361 |
1 files changed, 290 insertions, 71 deletions
diff --git a/drivers/gpu/mali/mali400ko/driver/src/devicedrv/mali/common/mali_kernel_mem_mmu.c b/drivers/gpu/mali/mali400ko/driver/src/devicedrv/mali/common/mali_kernel_mem_mmu.c index 7a48e27ede7..46eaea8eea5 100644 --- a/drivers/gpu/mali/mali400ko/driver/src/devicedrv/mali/common/mali_kernel_mem_mmu.c +++ b/drivers/gpu/mali/mali400ko/driver/src/devicedrv/mali/common/mali_kernel_mem_mmu.c @@ -18,6 +18,7 @@ #include "mali_kernel_mem_os.h" #include "mali_kernel_session_manager.h" #include "mali_kernel_core.h" +#include "mali_kernel_rendercore.h" #if defined USING_MALI400_L2_CACHE #include "mali_kernel_l2_cache.h" @@ -205,6 +206,7 @@ typedef struct mali_kernel_memory_mmu memory_session * active_session; /**< Active session, NULL if no session is active */ u32 usage_count; /**< Number of nested activations of the active session */ _mali_osk_list_t callbacks; /**< Callback registered for MMU idle notification */ + void *core; int in_page_fault_handler; @@ -329,10 +331,13 @@ static _mali_osk_errcode_t mali_memory_core_resource_os_memory(_mali_osk_resourc * also call this directly (depending on compilation options), having locked * the descriptor. * + * This function will fail if it is unable to put the MMU in stall mode (which + * might be the case if a page fault is also being processed). + * * @param args see _mali_uk_mem_munmap_s in "mali_uk_types.h" * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. */ -static void _mali_ukk_mem_munmap_internal( _mali_uk_mem_munmap_s *args ); +static _mali_osk_errcode_t _mali_ukk_mem_munmap_internal( _mali_uk_mem_munmap_s *args ); /** * The MMU interrupt handler @@ -373,6 +378,30 @@ static u32 mali_mmu_register_read(mali_kernel_memory_mmu * unit, mali_mmu_regist */ static void mali_mmu_register_write(mali_kernel_memory_mmu * unit, mali_mmu_register reg, u32 val); +/** + * Issues the reset command to the MMU and waits for HW to be ready again + * @param mmu The MMU to reset + */ +static void mali_mmu_raw_reset(mali_kernel_memory_mmu * mmu); + +/** + * Issues the enable paging command to the MMU and waits for HW to complete the request + * @param mmu The MMU to enable paging for + */ +static void mali_mmu_enable_paging(mali_kernel_memory_mmu * mmu); + +/** + * Issues the enable stall command to the MMU and waits for HW to complete the request + * @param mmu The MMU to enable paging for + * @return MALI_TRUE if HW stall was successfully engaged, otherwise MALI_FALSE (req timed out) + */ +static mali_bool mali_mmu_enable_stall(mali_kernel_memory_mmu * mmu); + +/** + * Issues the disable stall command to the MMU and waits for HW to complete the request + * @param mmu The MMU to enable paging for + */ +static void mali_mmu_disable_stall(mali_kernel_memory_mmu * mmu); #if MALI_USE_UNIFIED_MEMORY_PROVIDER != 0 static void ump_memory_release(void * ctx, void * handle); @@ -533,7 +562,7 @@ static void mali_memory_core_terminate(mali_kernel_subsystem_identifier id) _MALI_OSK_LIST_FOREACHENTRY(mmu, temp_mmu, &mmu_head, mali_kernel_memory_mmu, list) { /* reset to defaults */ - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_SOFT_RESET); + mali_mmu_raw_reset(mmu); /* unregister the irq */ _mali_osk_irq_term(mmu->irq); @@ -544,6 +573,7 @@ static void mali_memory_core_terminate(mali_kernel_subsystem_identifier id) /* release resources */ _mali_osk_mem_unmapioregion(mmu->base, mmu->mapping_size, mmu->mapped_registers); _mali_osk_mem_unreqregion(mmu->base, mmu->mapping_size); + _mali_osk_lock_term(mmu->lock); _mali_osk_free(mmu); } @@ -630,8 +660,9 @@ static _mali_osk_errcode_t mali_memory_core_session_begin(struct mali_session_da for (i = 0; i < MALI_MMU_PAGE_SIZE/4; i++) { /* mark each page table as not present */ - _mali_osk_mem_iowrite32(session_data->page_directory_mapped, sizeof(u32) * i, 0); + _mali_osk_mem_iowrite32_relaxed(session_data->page_directory_mapped, sizeof(u32) * i, 0); } + _mali_osk_write_mem_barrier(); /* page_table_mapped[] is already set to NULL by _mali_osk_calloc call */ @@ -813,8 +844,9 @@ static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data) for(i = 0; i < MALI_MMU_PAGE_SIZE/4; i++) { - _mali_osk_mem_iowrite32( mapping, i * sizeof(u32), data); + _mali_osk_mem_iowrite32_relaxed( mapping, i * sizeof(u32), data); } + _mali_osk_mem_barrier(); MALI_SUCCESS; } @@ -887,7 +919,7 @@ static _mali_osk_errcode_t mali_memory_core_load_complete(mali_kernel_subsystem_ _MALI_OSK_LIST_FOREACHENTRY(mmu, temp_mmu, &mmu_head, mali_kernel_memory_mmu, list) { mali_mmu_register_write(mmu, MALI_MMU_REGISTER_DTE_ADDR, mali_empty_page_directory); - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING); + mali_mmu_enable_paging(mmu); } MALI_DEBUG_PRINT(4, ("MMUs activated\n")); @@ -992,7 +1024,7 @@ static _mali_osk_errcode_t mali_memory_core_resource_mmu(_mali_osk_resource_t * /* setup MMU interrupt mask */ /* set all values to known defaults */ - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_SOFT_RESET); + mali_mmu_raw_reset(mmu); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); /* setup MMU page directory pointer */ /* The mali_page_directory pointer is guaranteed to be 4kb aligned because we've used get_zeroed_page to accquire it */ @@ -1021,7 +1053,7 @@ static _mali_osk_errcode_t mali_memory_core_resource_mmu(_mali_osk_resource_t * } /* set to a known state */ - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_SOFT_RESET); + mali_mmu_raw_reset(mmu); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); MALI_DEBUG_PRINT(2, ("MMU registered\n")); @@ -1157,6 +1189,7 @@ static _mali_osk_errcode_t mali_kernel_memory_mmu_interrupt_handler_upper_half(v { mali_kernel_memory_mmu * mmu; u32 int_stat; + mali_core_renderunit *core; if (mali_benchmark) MALI_SUCCESS; @@ -1164,15 +1197,19 @@ static _mali_osk_errcode_t mali_kernel_memory_mmu_interrupt_handler_upper_half(v MALI_DEBUG_ASSERT_POINTER(mmu); + core = (mali_core_renderunit *)mmu->core; + if(core && (CORE_OFF == core->state)) + { + MALI_SUCCESS; + } + /* check if it was our device which caused the interrupt (we could be sharing the IRQ line) */ int_stat = mali_mmu_register_read(mmu, MALI_MMU_REGISTER_INT_STATUS); if (0 == int_stat) { - MALI_DEBUG_PRINT(5, ("Ignoring shared interrupt\n")); MALI_ERROR(_MALI_OSK_ERR_FAULT); /* no bits set, we are sharing the IRQ line and someone else caused the interrupt */ } - MALI_DEBUG_PRINT(1, ("mali_kernel_memory_mmu_interrupt_handler_upper_half\n")); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_INT_MASK, 0); @@ -1180,13 +1217,10 @@ static _mali_osk_errcode_t mali_kernel_memory_mmu_interrupt_handler_upper_half(v if (int_stat & MALI_MMU_INTERRUPT_PAGE_FAULT) { - MALI_PRINT(("Page fault on %s\n", mmu->description)); - _mali_osk_irq_schedulework(mmu->irq); } if (int_stat & MALI_MMU_INTERRUPT_READ_BUS_ERROR) { - MALI_PRINT(("Bus read error on %s\n", mmu->description)); /* clear interrupt flag */ mali_mmu_register_write(mmu, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_READ_BUS_ERROR); /* reenable it */ @@ -1219,6 +1253,9 @@ static void mali_kernel_mmu_bus_reset(mali_kernel_memory_mmu * mmu) /* no new request will come from any of the connected cores from now * we must now flush the playback buffer for any requests queued already */ + + _mali_osk_lock_wait(mmu->lock, _MALI_OSK_LOCKMODE_RW); + MALI_DEBUG_PRINT(4, ("Switching to the special page fault flush page directory\n")); /* don't use the mali_mmu_activate_address_space function here as we can't stall the MMU */ mali_mmu_register_write(mmu, MALI_MMU_REGISTER_DTE_ADDR, mali_page_fault_flush_page_directory); @@ -1239,16 +1276,23 @@ static void mali_kernel_mmu_bus_reset(mali_kernel_memory_mmu * mmu) MALI_DEBUG_PRINT_IF(1, i == replay_buffer_max_number_of_checks, ("MMU: %s: Failed to flush replay buffer on page fault\n", mmu->description)); MALI_DEBUG_PRINT(1, ("Replay playback took %ld usec\n", i * replay_buffer_check_interval)); } + + _mali_osk_lock_signal(mmu->lock, _MALI_OSK_LOCKMODE_RW); + #endif /* notify all subsystems that the core should be reset once the bus is actually stopped */ MALI_DEBUG_PRINT(4,("Sending job abort command to subsystems\n")); _mali_kernel_core_broadcast_subsystem_message(MMU_KILL_STEP2_RESET_ALL_CORES_AND_ABORT_THEIR_JOBS, (u32)mmu); + _mali_osk_lock_wait(mmu->lock, _MALI_OSK_LOCKMODE_RW); + /* reprogram the MMU */ - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_SOFT_RESET); + mali_mmu_raw_reset(mmu); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_DTE_ADDR, mali_empty_page_directory); /* no session is active, so just activate the empty page directory */ - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING); + mali_mmu_enable_paging(mmu); + + _mali_osk_lock_signal(mmu->lock, _MALI_OSK_LOCKMODE_RW); /* release the extra address space reference, will schedule */ mali_memory_core_mmu_release_address_space_reference(mmu); @@ -1258,6 +1302,101 @@ static void mali_kernel_mmu_bus_reset(mali_kernel_memory_mmu * mmu) MALI_DEBUG_PRINT(4, ("Page fault handling complete\n")); } +static void mali_mmu_raw_reset(mali_kernel_memory_mmu * mmu) +{ + const int max_loop_count = 100; + const int delay_in_usecs = 1; + + mali_mmu_register_write(mmu, MALI_MMU_REGISTER_DTE_ADDR, 0xCAFEBABE); + mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_SOFT_RESET); + + if (!mali_benchmark) + { + int i; + for (i = 0; i < max_loop_count; ++i) + { + if (mali_mmu_register_read(mmu, MALI_MMU_REGISTER_DTE_ADDR) == 0) + { + break; + } + _mali_osk_time_ubusydelay(delay_in_usecs); + } + MALI_DEBUG_PRINT_IF(1, (max_loop_count == i), ("Reset request failed, MMU status is 0x%08X\n", mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS))); + } +} + +static void mali_mmu_enable_paging(mali_kernel_memory_mmu * mmu) +{ + const int max_loop_count = 100; + const int delay_in_usecs = 1; + + mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING); + + if (!mali_benchmark) + { + int i; + for (i = 0; i < max_loop_count; ++i) + { + if (mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_PAGING_ENABLED) + { + break; + } + _mali_osk_time_ubusydelay(delay_in_usecs); + } + MALI_DEBUG_PRINT_IF(1, (max_loop_count == i), ("Enable paging request failed, MMU status is 0x%08X\n", mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS))); + } +} + +static mali_bool mali_mmu_enable_stall(mali_kernel_memory_mmu * mmu) +{ + const int max_loop_count = 100; + const int delay_in_usecs = 999; + int i; + + mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL); + + if (!mali_benchmark) + { + for (i = 0; i < max_loop_count; ++i) + { + if (mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_STALL_ACTIVE) + { + break; + } + _mali_osk_time_ubusydelay(delay_in_usecs); + } + MALI_DEBUG_PRINT_IF(1, (max_loop_count == i), ("Enable stall request failed, MMU status is 0x%08X\n", mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS))); + if (max_loop_count == i) + { + return MALI_FALSE; + } + } + + return MALI_TRUE; +} + +static void mali_mmu_disable_stall(mali_kernel_memory_mmu * mmu) +{ + const int max_loop_count = 100; + const int delay_in_usecs = 1; + int i; + + mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL); + + if (!mali_benchmark) + { + for (i = 0; i < max_loop_count; ++i) + { + if ((mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_STALL_ACTIVE) == 0) + { + break; + } + _mali_osk_time_ubusydelay(delay_in_usecs); + } + MALI_DEBUG_PRINT_IF(1, (max_loop_count == i), ("Disable stall request failed, MMU status is 0x%08X\n", mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS))); + } +} + void mali_kernel_mmu_reset(void * input_mmu) { mali_kernel_memory_mmu * mmu; @@ -1273,10 +1412,10 @@ void mali_kernel_mmu_reset(void * input_mmu) return; } _mali_osk_lock_wait(mmu->lock, _MALI_OSK_LOCKMODE_RW); - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_SOFT_RESET); + mali_mmu_raw_reset(mmu); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_DTE_ADDR, mali_empty_page_directory); /* no session is active, so just activate the empty page directory */ - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING); + mali_mmu_enable_paging(mmu); _mali_osk_lock_signal(mmu->lock, _MALI_OSK_LOCKMODE_RW); } @@ -1302,6 +1441,7 @@ static void mali_kernel_memory_mmu_interrupt_handler_bottom_half(void * data) { mali_kernel_memory_mmu * mmu; u32 raw, fault_address, status; + mali_core_renderunit *core; if (NULL == data) { @@ -1310,11 +1450,19 @@ static void mali_kernel_memory_mmu_interrupt_handler_bottom_half(void * data) } mmu = (mali_kernel_memory_mmu*)data; - MALI_DEBUG_PRINT(4, ("Locking subsystems\n")); /* lock all subsystems */ _mali_kernel_core_broadcast_subsystem_message(MMU_KILL_STEP0_LOCK_SUBSYSTEM, (u32)mmu); + /* Pointer to core holding this MMU */ + core = (mali_core_renderunit *)mmu->core; + + if(CORE_OFF == core->state) + { + _mali_kernel_core_broadcast_subsystem_message(MMU_KILL_STEP4_UNLOCK_SUBSYSTEM, (u32)mmu); + return; + } + raw = mali_mmu_register_read(mmu, MALI_MMU_REGISTER_INT_RAWSTAT); status = mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS); @@ -1404,7 +1552,6 @@ static void mali_mmu_register_write(mali_kernel_memory_mmu * unit, mali_mmu_regi _mali_osk_mem_iowrite32(unit->mapped_registers, (u32)reg * sizeof(u32), val); } - #if MALI_USE_UNIFIED_MEMORY_PROVIDER != 0 static mali_physical_memory_allocation_result ump_memory_commit(void* ctx, mali_allocation_engine * engine, mali_memory_allocation * descriptor, u32* offset, mali_physical_memory_allocation * alloc_info) { @@ -1575,6 +1722,7 @@ _mali_osk_errcode_t _mali_ukk_attach_ump_mem( _mali_uk_attach_ump_mem_s *args ) descriptor->mali_address = args->mali_address; descriptor->mali_addr_mapping_info = (void*)session_data; descriptor->process_addr_mapping_info = NULL; /* do not map to process address space */ + descriptor->lock = session_data->lock; if (args->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { descriptor->flags = MALI_MEMORY_ALLOCATION_FLAG_MAP_GUARD_PAGE; @@ -1594,14 +1742,19 @@ _mali_osk_errcode_t _mali_ukk_attach_ump_mem( _mali_uk_attach_ump_mem_s *args ) external_memory_allocator.name = "UMP Memory"; external_memory_allocator.next = NULL; + _mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW); + if (_MALI_OSK_ERR_OK != mali_allocation_engine_allocate_memory(memory_engine, descriptor, &external_memory_allocator, NULL)) { + _mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW); mali_descriptor_mapping_free(session_data->descriptor_mapping, md); ump_dd_reference_release(ump_mem); _mali_osk_free(descriptor); MALI_ERROR(_MALI_OSK_ERR_NOMEM); } + _mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW); + args->cookie = md; MALI_DEBUG_PRINT(5,("Returning from UMP attach\n")); @@ -1629,7 +1782,13 @@ _mali_osk_errcode_t _mali_ukk_release_ump_mem( _mali_uk_release_ump_mem_s *args } mali_descriptor_mapping_free(session_data->descriptor_mapping, args->cookie); + + _mali_osk_lock_wait( session_data->lock, _MALI_OSK_LOCKMODE_RW ); + mali_allocation_engine_release_memory(memory_engine, descriptor); + + _mali_osk_lock_signal( session_data->lock, _MALI_OSK_LOCKMODE_RW ); + _mali_osk_free(descriptor); MALI_SUCCESS; @@ -1768,7 +1927,7 @@ _mali_osk_errcode_t _mali_ukk_map_external_mem( _mali_uk_map_external_mem_s *arg descriptor->mali_address = args->mali_address; descriptor->mali_addr_mapping_info = (void*)session_data; descriptor->process_addr_mapping_info = NULL; /* do not map to process address space */ - descriptor->lock = NULL; + descriptor->lock = session_data->lock; if (args->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { descriptor->flags = MALI_MEMORY_ALLOCATION_FLAG_MAP_GUARD_PAGE; @@ -1781,13 +1940,18 @@ _mali_osk_errcode_t _mali_ukk_map_external_mem( _mali_uk_map_external_mem_s *arg MALI_ERROR(_MALI_OSK_ERR_FAULT); } + _mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW); + if (_MALI_OSK_ERR_OK != mali_allocation_engine_allocate_memory(memory_engine, descriptor, &external_memory_allocator, NULL)) { + _mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW); mali_descriptor_mapping_free(session_data->descriptor_mapping, md); _mali_osk_free(descriptor); MALI_ERROR(_MALI_OSK_ERR_NOMEM); } + _mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW); + args->cookie = md; MALI_DEBUG_PRINT(5,("Returning from range_map_external_memory\n")); @@ -1815,7 +1979,13 @@ _mali_osk_errcode_t _mali_ukk_unmap_external_mem( _mali_uk_unmap_external_mem_s } mali_descriptor_mapping_free(session_data->descriptor_mapping, args->cookie); + + _mali_osk_lock_wait( session_data->lock, _MALI_OSK_LOCKMODE_RW ); + mali_allocation_engine_release_memory(memory_engine, descriptor); + + _mali_osk_lock_signal( session_data->lock, _MALI_OSK_LOCKMODE_RW ); + _mali_osk_free(descriptor); MALI_SUCCESS; @@ -1870,6 +2040,8 @@ void mali_mmu_page_table_cache_destroy(void) _mali_osk_free(alloc->usage_map); _mali_osk_free(alloc); } + + _mali_osk_lock_term(page_table_cache.lock); } _mali_osk_errcode_t mali_mmu_get_table_page(u32 *table_page, mali_io_address *mapping) @@ -1941,7 +2113,14 @@ _mali_osk_errcode_t mali_mmu_get_table_page(u32 *table_page, mali_io_address *ma _mali_osk_set_nonatomic_bit(0, alloc->usage_map); - _mali_osk_list_add(&alloc->list, &page_table_cache.partial); + if (alloc->num_pages > 1) + { + _mali_osk_list_add(&alloc->list, &page_table_cache.partial); + } + else + { + _mali_osk_list_add(&alloc->list, &page_table_cache.full); + } _mali_osk_lock_signal(page_table_cache.lock, _MALI_OSK_LOCKMODE_RW); *table_page = alloc->pages.phys_base; /* return the first page */ @@ -2001,8 +2180,21 @@ void mali_mmu_release_table_page(u32 pa) _mali_osk_memset((void*)( ((u32)alloc->pages.mapping) + (pa - start) ), 0, MALI_MMU_PAGE_SIZE); - /* transfer to partial list */ - _mali_osk_list_move(&alloc->list, &page_table_cache.partial); + + if (0 == alloc->usage_count) + { + /* empty, release whole page alloc */ + _mali_osk_list_del(&alloc->list); + alloc->pages.release(&alloc->pages); + _mali_osk_free(alloc->usage_map); + _mali_osk_free(alloc); + } + else + { + /* transfer to partial list */ + _mali_osk_list_move(&alloc->list, &page_table_cache.partial); + } + _mali_osk_lock_signal(page_table_cache.lock, _MALI_OSK_LOCKMODE_RW); MALI_DEBUG_PRINT(4, ("(full list)Released table page 0x%08X to the cache\n", pa)); return; @@ -2014,6 +2206,17 @@ void mali_mmu_release_table_page(u32 pa) _mali_osk_lock_signal(page_table_cache.lock, _MALI_OSK_LOCKMODE_RW); } +void mali_memory_core_mmu_owner(void *core, void *mmu_ptr) +{ + mali_kernel_memory_mmu *mmu; + + MALI_DEBUG_ASSERT_POINTER(mmu_ptr); + MALI_DEBUG_ASSERT_POINTER(core); + + mmu = (mali_kernel_memory_mmu *)mmu_ptr; + mmu->core = core; +} + void* mali_memory_core_mmu_lookup(u32 id) { mali_kernel_memory_mmu * mmu, * temp_mmu; @@ -2030,24 +2233,10 @@ void* mali_memory_core_mmu_lookup(u32 id) void mali_mmu_activate_address_space(mali_kernel_memory_mmu * mmu, u32 page_directory) { - const int delay_in_usecs = 10; - const int max_loop_count = 10; - int i; - - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL); - - if (!mali_benchmark) { - for (i = 0; i < max_loop_count; ++i) - { - if (mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_STALL_ACTIVE) break; - _mali_osk_time_ubusydelay(delay_in_usecs); - } - MALI_DEBUG_PRINT_IF(1, (max_loop_count == i), ("Stall request failed, swapping anyway\n")); - } - + mali_mmu_enable_stall(mmu); /* this might fail, but changing the DTE address and ZAP should work anyway... */ mali_mmu_register_write(mmu, MALI_MMU_REGISTER_DTE_ADDR, page_directory); mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL); + mali_mmu_disable_stall(mmu); } _mali_osk_errcode_t mali_memory_core_mmu_activate_page_table(void* mmu_ptr, struct mali_session_data * mali_session_data, void(*callback)(void*), void * callback_argument) @@ -2517,8 +2706,9 @@ static _mali_osk_errcode_t mali_address_manager_map(mali_memory_allocation * des for ( ; mali_address < mali_address_end; mali_address += MALI_MMU_PAGE_SIZE, current_phys_addr += MALI_MMU_PAGE_SIZE) { MALI_DEBUG_ASSERT_POINTER(session_data->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]); - _mali_osk_mem_iowrite32(session_data->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)], MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32), current_phys_addr | MALI_MMU_FLAGS_WRITE_PERMISSION | MALI_MMU_FLAGS_READ_PERMISSION | MALI_MMU_FLAGS_PRESENT); + _mali_osk_mem_iowrite32_relaxed(session_data->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)], MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32), current_phys_addr | MALI_MMU_FLAGS_WRITE_PERMISSION | MALI_MMU_FLAGS_READ_PERMISSION | MALI_MMU_FLAGS_PRESENT); } + _mali_osk_write_mem_barrier(); #if defined USING_MALI400_L2_CACHE if (1 == has_active_mmus) @@ -2574,7 +2764,7 @@ _mali_osk_errcode_t _mali_ukk_mem_mmap( _mali_uk_mem_mmap_s *args ) descriptor->lock = session_data->lock; _mali_osk_list_init( &descriptor->list ); - _mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW); + _mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW); if (0 == mali_allocation_engine_allocate_memory(memory_engine, descriptor, physical_memory_allocators, &session_data->memory_head)) { @@ -2587,12 +2777,9 @@ _mali_osk_errcode_t _mali_ukk_mem_mmap( _mali_uk_mem_mmap_s *args ) _mali_osk_lock_wait(mmu->lock, _MALI_OSK_LOCKMODE_RW); - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL); - if (!mali_benchmark) { - while ( (mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_STALL_ACTIVE) == 0) _mali_osk_time_ubusydelay(1); - } + mali_mmu_enable_stall(mmu); /* this might fail, but ZAP should work anyway... */ mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL); + mali_mmu_disable_stall(mmu); _mali_osk_lock_signal(mmu->lock, _MALI_OSK_LOCKMODE_RW); } @@ -2618,7 +2805,7 @@ _mali_osk_errcode_t _mali_ukk_mem_mmap( _mali_uk_mem_mmap_s *args ) } } -static void _mali_ukk_mem_munmap_internal( _mali_uk_mem_munmap_s *args ) +static _mali_osk_errcode_t _mali_ukk_mem_munmap_internal( _mali_uk_mem_munmap_s *args ) { memory_session * session_data; mali_kernel_memory_mmu * mmu, * temp_mmu; @@ -2641,24 +2828,34 @@ static void _mali_ukk_mem_munmap_internal( _mali_uk_mem_munmap_s *args ) */ _MALI_OSK_LIST_FOREACHENTRY(mmu, temp_mmu, &session_data->active_mmus, mali_kernel_memory_mmu, session_link) { - const int max_loop_count = 100; - const int sleep_duration = 1; /* must be below 1000 */ - int i; - + u32 status; + status = mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS); + if ( MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE == (status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) ) { + MALI_DEBUG_PRINT(2, ("Stopped stall attempt for mmu with id %d since it is in page fault mode.\n", mmu->id)); + continue; + } _mali_osk_lock_wait(mmu->lock, _MALI_OSK_LOCKMODE_RW); - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL); - - if (!mali_benchmark) + /* + * If we're unable to stall, then make sure we tell our caller that, + * the caller should then release the session lock for a while, + * then this function again. + * This function will fail if we're in page fault mode, and to get + * out of page fault mode, the page fault handler must be able to + * take the session lock. + */ + if (!mali_mmu_enable_stall(mmu)) { - for ( i = 0; i < max_loop_count; i++) - { - if (mali_mmu_register_read(mmu, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_STALL_ACTIVE) break; - _mali_osk_time_ubusydelay(sleep_duration); - } - - MALI_DEBUG_PRINT_IF(3, max_loop_count == i, ("Stall failed, trying zap anyway\n")); + _mali_osk_lock_signal(mmu->lock, _MALI_OSK_LOCKMODE_RW); + return _MALI_OSK_ERR_BUSY; } + + _mali_osk_lock_signal(mmu->lock, _MALI_OSK_LOCKMODE_RW); + } + + _MALI_OSK_LIST_FOREACHENTRY(mmu, temp_mmu, &session_data->active_mmus, mali_kernel_memory_mmu, session_link) + { + _mali_osk_lock_wait(mmu->lock, _MALI_OSK_LOCKMODE_RW); } /* This function also removes the memory from the session's memory list */ @@ -2671,10 +2868,12 @@ static void _mali_ukk_mem_munmap_internal( _mali_uk_mem_munmap_s *args ) _MALI_OSK_LIST_FOREACHENTRY(mmu, temp_mmu, &session_data->active_mmus, mali_kernel_memory_mmu, session_link) { mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); - mali_mmu_register_write(mmu, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL); + mali_mmu_disable_stall(mmu); _mali_osk_lock_signal(mmu->lock, _MALI_OSK_LOCKMODE_RW); } + + return _MALI_OSK_ERR_OK; } /* Handler for unmapping memory for MMU builds */ @@ -2682,6 +2881,7 @@ _mali_osk_errcode_t _mali_ukk_mem_munmap( _mali_uk_mem_munmap_s *args ) { mali_memory_allocation * descriptor; _mali_osk_lock_t *descriptor_lock; + _mali_osk_errcode_t err; descriptor = (mali_memory_allocation *)args->cookie; MALI_DEBUG_ASSERT_POINTER(descriptor); @@ -2695,20 +2895,34 @@ _mali_osk_errcode_t _mali_ukk_mem_munmap( _mali_uk_mem_munmap_s *args ) descriptor_lock = descriptor->lock; /* should point to the session data lock... */ - if (descriptor_lock) + err = _MALI_OSK_ERR_BUSY; + while (err == _MALI_OSK_ERR_BUSY) { - _mali_osk_lock_wait( descriptor_lock, _MALI_OSK_LOCKMODE_RW ); - } - /* Noninterruptable spinlock type, so must always have locked. Checking should've been done in OSK function. */ + if (descriptor_lock) + { + _mali_osk_lock_wait( descriptor_lock, _MALI_OSK_LOCKMODE_RW ); + } - _mali_ukk_mem_munmap_internal( args ); - /* descriptor is no longer valid - it may've been freed */ + err = _mali_ukk_mem_munmap_internal( args ); - if (descriptor_lock) - { - _mali_osk_lock_signal( descriptor_lock, _MALI_OSK_LOCKMODE_RW ); + if (descriptor_lock) + { + _mali_osk_lock_signal( descriptor_lock, _MALI_OSK_LOCKMODE_RW ); + } + + if (err == _MALI_OSK_ERR_BUSY) + { + /* + * Reason for this; + * We where unable to stall the MMU, probably because we are in page fault handling. + * Sleep for a while with the session lock released, then try again. + * Abnormal termination of programs with running Mali jobs is a normal reason for this. + */ + _mali_osk_time_ubusydelay(10); + } } - return _MALI_OSK_ERR_OK; + + return err; } /* Is called when the rendercore wants the mmu to give an interrupt */ @@ -2922,3 +3136,8 @@ _mali_osk_errcode_t _mali_ukk_free_big_block( _mali_uk_free_big_block_s *args ) MALI_IGNORE( args ); return _MALI_OSK_ERR_FAULT; } + +u32 _mali_ukk_report_memory_usage(void) +{ + return mali_allocation_engine_memory_usage(physical_memory_allocators); +} |