From 673a618f0fa0da41cf25897d40edbb0293b4765d Mon Sep 17 00:00:00 2001 From: Jeff Preshing Date: Mon, 1 Feb 2016 08:09:52 -0500 Subject: [PATCH] Initial commit --- .clang-format | 9 + .gitignore | 3 + CMakeLists.txt | 114 +++ LICENSE | 22 + README.md | 91 ++ cmake/AddSample.cmake | 37 + cmake/JunctionProjectDefs.cmake | 17 + cmake/junction_config.h.in | 11 + cmake/modules/FindCDS.cmake | 25 + cmake/modules/FindFolly.cmake | 9 + cmake/modules/FindNBDS.cmake | 37 + cmake/modules/FindTBB.cmake | 221 +++++ cmake/modules/FindTervel.cmake | 17 + cmake/modules/FindTurf.cmake | 27 + docs/cmake-gui.png | Bin 0 -> 24311 bytes docs/vs-solution.png | Bin 0 -> 11481 bytes junction/Averager.cpp | 30 + junction/Averager.h | 63 ++ junction/ConcurrentMap_Grampa.cpp | 47 + junction/ConcurrentMap_Grampa.h | 571 ++++++++++++ junction/ConcurrentMap_LeapFrog.cpp | 37 + junction/ConcurrentMap_LeapFrog.h | 336 +++++++ junction/ConcurrentMap_Linear.cpp | 38 + junction/ConcurrentMap_Linear.h | 358 +++++++ junction/Core.h | 33 + junction/MapTraits.h | 44 + junction/QSBR.cpp | 107 +++ junction/QSBR.h | 91 ++ junction/SimpleJobCoordinator.h | 92 ++ junction/SingleMap_Linear.h | 207 +++++ junction/details/Grampa.cpp | 65 ++ junction/details/Grampa.h | 875 ++++++++++++++++++ junction/details/LeapFrog.cpp | 57 ++ junction/details/LeapFrog.h | 561 +++++++++++ junction/details/Linear.cpp | 46 + junction/details/Linear.h | 375 ++++++++ junction/extra/MapAdapter.h | 24 + junction/extra/MemHook_NBDS.cpp | 29 + junction/extra/MemHook_TBB.cpp | 32 + junction/extra/impl/MapAdapter_CDS_Cuckoo.h | 112 +++ junction/extra/impl/MapAdapter_CDS_Michael.h | 115 +++ junction/extra/impl/MapAdapter_Folly.h | 79 ++ junction/extra/impl/MapAdapter_Grampa.h | 62 ++ junction/extra/impl/MapAdapter_LeapFrog.h | 62 ++ junction/extra/impl/MapAdapter_Linear.h | 62 ++ junction/extra/impl/MapAdapter_Linear_Mutex.h | 79 ++ .../extra/impl/MapAdapter_Linear_RWLock.h | 79 ++ junction/extra/impl/MapAdapter_NBDS.h | 93 ++ junction/extra/impl/MapAdapter_Null.h | 67 ++ junction/extra/impl/MapAdapter_StdMap.h | 106 +++ junction/extra/impl/MapAdapter_TBB.h | 82 ++ junction/extra/impl/MapAdapter_Tervel.h | 90 ++ junction/striped/AutoResetEvent.h | 55 ++ junction/striped/ConditionBank.cpp | 37 + junction/striped/ConditionBank.h | 67 ++ junction/striped/ConditionPair.h | 31 + junction/striped/ManualResetEvent.h | 99 ++ junction/striped/Mutex.h | 76 ++ samples/MallocTest/CMakeLists.txt | 10 + samples/MallocTest/MallocTest.cpp | 35 + samples/MapCorrectnessTests/CMakeLists.txt | 12 + .../MapCorrectnessTests.cpp | 49 + samples/MapCorrectnessTests/TestChurn.h | 173 ++++ samples/MapCorrectnessTests/TestEnvironment.h | 43 + .../TestInsertDifferentKeys.h | 134 +++ .../MapCorrectnessTests/TestInsertSameKeys.h | 130 +++ .../junction_userconfig.h.in | 4 + samples/MapMemoryBench/CMakeLists.txt | 10 + samples/MapMemoryBench/MapMemoryBench.cpp | 71 ++ samples/MapMemoryBench/RenderGraphs.py | 193 ++++ samples/MapMemoryBench/TestAllMaps.py | 54 ++ samples/MapPerformanceTests/CMakeLists.txt | 10 + .../MapPerformanceTests.cpp | 301 ++++++ samples/MapPerformanceTests/RenderGraphs.py | 301 ++++++ samples/MapPerformanceTests/TestAllMaps.py | 54 ++ samples/MapScalabilityTests/CMakeLists.txt | 10 + .../MapScalabilityTests.cpp | 278 ++++++ samples/MapScalabilityTests/RenderGraphs.py | 282 ++++++ samples/MapScalabilityTests/TestAllMaps.py | 54 ++ 79 files changed, 8419 insertions(+) create mode 100644 .clang-format create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 README.md create mode 100644 cmake/AddSample.cmake create mode 100644 cmake/JunctionProjectDefs.cmake create mode 100644 cmake/junction_config.h.in create mode 100644 cmake/modules/FindCDS.cmake create mode 100644 cmake/modules/FindFolly.cmake create mode 100644 cmake/modules/FindNBDS.cmake create mode 100644 cmake/modules/FindTBB.cmake create mode 100644 cmake/modules/FindTervel.cmake create mode 100644 cmake/modules/FindTurf.cmake create mode 100644 docs/cmake-gui.png create mode 100644 docs/vs-solution.png create mode 100644 junction/Averager.cpp create mode 100644 junction/Averager.h create mode 100644 junction/ConcurrentMap_Grampa.cpp create mode 100644 junction/ConcurrentMap_Grampa.h create mode 100644 junction/ConcurrentMap_LeapFrog.cpp create mode 100644 junction/ConcurrentMap_LeapFrog.h create mode 100644 junction/ConcurrentMap_Linear.cpp create mode 100644 junction/ConcurrentMap_Linear.h create mode 100644 junction/Core.h create mode 100644 junction/MapTraits.h create mode 100644 junction/QSBR.cpp create mode 100644 junction/QSBR.h create mode 100644 junction/SimpleJobCoordinator.h create mode 100644 junction/SingleMap_Linear.h create mode 100644 junction/details/Grampa.cpp create mode 100644 junction/details/Grampa.h create mode 100644 junction/details/LeapFrog.cpp create mode 100644 junction/details/LeapFrog.h create mode 100644 junction/details/Linear.cpp create mode 100644 junction/details/Linear.h create mode 100644 junction/extra/MapAdapter.h create mode 100644 junction/extra/MemHook_NBDS.cpp create mode 100644 junction/extra/MemHook_TBB.cpp create mode 100644 junction/extra/impl/MapAdapter_CDS_Cuckoo.h create mode 100644 junction/extra/impl/MapAdapter_CDS_Michael.h create mode 100644 junction/extra/impl/MapAdapter_Folly.h create mode 100644 junction/extra/impl/MapAdapter_Grampa.h create mode 100644 junction/extra/impl/MapAdapter_LeapFrog.h create mode 100644 junction/extra/impl/MapAdapter_Linear.h create mode 100644 junction/extra/impl/MapAdapter_Linear_Mutex.h create mode 100644 junction/extra/impl/MapAdapter_Linear_RWLock.h create mode 100644 junction/extra/impl/MapAdapter_NBDS.h create mode 100644 junction/extra/impl/MapAdapter_Null.h create mode 100644 junction/extra/impl/MapAdapter_StdMap.h create mode 100644 junction/extra/impl/MapAdapter_TBB.h create mode 100644 junction/extra/impl/MapAdapter_Tervel.h create mode 100644 junction/striped/AutoResetEvent.h create mode 100644 junction/striped/ConditionBank.cpp create mode 100644 junction/striped/ConditionBank.h create mode 100644 junction/striped/ConditionPair.h create mode 100644 junction/striped/ManualResetEvent.h create mode 100644 junction/striped/Mutex.h create mode 100644 samples/MallocTest/CMakeLists.txt create mode 100644 samples/MallocTest/MallocTest.cpp create mode 100644 samples/MapCorrectnessTests/CMakeLists.txt create mode 100644 samples/MapCorrectnessTests/MapCorrectnessTests.cpp create mode 100644 samples/MapCorrectnessTests/TestChurn.h create mode 100644 samples/MapCorrectnessTests/TestEnvironment.h create mode 100644 samples/MapCorrectnessTests/TestInsertDifferentKeys.h create mode 100644 samples/MapCorrectnessTests/TestInsertSameKeys.h create mode 100644 samples/MapCorrectnessTests/junction_userconfig.h.in create mode 100644 samples/MapMemoryBench/CMakeLists.txt create mode 100644 samples/MapMemoryBench/MapMemoryBench.cpp create mode 100644 samples/MapMemoryBench/RenderGraphs.py create mode 100644 samples/MapMemoryBench/TestAllMaps.py create mode 100644 samples/MapPerformanceTests/CMakeLists.txt create mode 100644 samples/MapPerformanceTests/MapPerformanceTests.cpp create mode 100644 samples/MapPerformanceTests/RenderGraphs.py create mode 100644 samples/MapPerformanceTests/TestAllMaps.py create mode 100644 samples/MapScalabilityTests/CMakeLists.txt create mode 100644 samples/MapScalabilityTests/MapScalabilityTests.cpp create mode 100644 samples/MapScalabilityTests/RenderGraphs.py create mode 100644 samples/MapScalabilityTests/TestAllMaps.py diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..be640a0 --- /dev/null +++ b/.clang-format @@ -0,0 +1,9 @@ +ColumnLimit: 130 +IndentWidth: 4 +UseTab: Never +AccessModifierOffset: -4 +NamespaceIndentation: None +DerivePointerAlignment: false +PointerAlignment: Left +SpaceAfterCStyleCast: true +AllowShortFunctionsOnASingleLine: None diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c10c08 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*build*/ +CMakeLists.txt.user +*~ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..0452094 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,114 @@ +cmake_minimum_required(VERSION 2.8.5) + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # If this is the root project, issue a project() command so that + # the Visual Studio generator will create an .sln file. + set(CMAKE_CONFIGURATION_TYPES "Debug;RelWithAsserts;RelWithDebInfo" CACHE INTERNAL "Build configs") + project(Junction) + set_property(GLOBAL PROPERTY USE_FOLDERS ON) + include(cmake/JunctionProjectDefs.cmake) + ApplyTurfBuildSettings() +elseif(NOT JUNCTION_FOUND) + message(FATAL_ERROR "You must include cmake/JunctionProjectDefs.cmake before adding this subdirectory") +endif() + +# Default values, can be overridden by user +set(JUNCTION_USERCONFIG "" CACHE STRING "Optional path to additional config file (relative to root CMakeLists.txt)") +set(JUNCTION_WITH_FOLLY FALSE CACHE BOOL "Use Folly") +set(JUNCTION_WITH_CDS FALSE CACHE BOOL "Use CDS") +set(JUNCTION_WITH_NBDS FALSE CACHE BOOL "Use NBDS") +set(JUNCTION_WITH_TBB FALSE CACHE BOOL "Use TBB") +set(JUNCTION_WITH_TERVEL FALSE CACHE BOOL "Use Tervel") +set(JUNCTION_TRACK_GRAMPA_STATS FALSE CACHE BOOL "Enable stats in ConcurrentMap_Grampa") +set(JUNCTION_USE_STRIPING TRUE CACHE BOOL "Allocate a fixed-size ConditionBank for striped primitives") + +# Initialize variables used to collect include dirs/libraries. +set(JUNCTION_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/include") +set(JUNCTION_ALL_INCLUDE_DIRS "${JUNCTION_INCLUDE_DIRS}") +set(JUNCTION_ALL_LIBRARIES junction) +set(JUNCTION_ALL_DLLS "") + +# Add turf targets +add_subdirectory(${TURF_ROOT} turf) +list(APPEND JUNCTION_ALL_INCLUDE_DIRS ${TURF_INCLUDE_DIRS}) +list(APPEND JUNCTION_ALL_LIBRARIES ${TURF_ALL_LIBRARIES}) + +# Optional: Locate Folly and append it to the list of include dirs/libraries. +if(JUNCTION_WITH_FOLLY) + find_package(Folly REQUIRED) + list(APPEND JUNCTION_ALL_INCLUDE_DIRS ${FOLLY_INCLUDE_DIR}) + list(APPEND JUNCTION_ALL_LIBRARIES ${FOLLY_LIBRARIES}) +endif() + +# Optional: Locate CDS and append it to the list of include dirs/libraries. +if(JUNCTION_WITH_CDS) + find_package(CDS REQUIRED) + list(APPEND JUNCTION_ALL_INCLUDE_DIRS ${CDS_INCLUDE_DIR}) + list(APPEND JUNCTION_ALL_LIBRARIES ${CDS_LIBRARY}) + list(APPEND JUNCTION_ALL_DLLS ${CDS_DLL}) +endif() + +# Optional: Locate NBDS and append it to the list of include dirs/libraries. +if(JUNCTION_WITH_NBDS) + set(NBDS_USE_TURF_HEAP FALSE CACHE BOOL "Redirect NBDS's memory allocator to use Turf") + find_package(NBDS REQUIRED) + list(APPEND JUNCTION_ALL_INCLUDE_DIRS ${NBDS_INCLUDE_DIRS}) + # If NBDS_USE_TURF_HEAP=1, then NBDS has dependencies on junction, so add junction to the linker + # command line again as needed by the GNU linker. + list(APPEND JUNCTION_ALL_LIBRARIES ${NBDS_LIBRARIES} junction) +endif() + +# Optional: Locate Intel TBB and append it to the list of include dirs/libraries. +if(JUNCTION_WITH_TBB) + set(TBB_USE_TURF_HEAP FALSE CACHE BOOL "Redirect TBB's memory allocator to use Turf") + find_package(TBB REQUIRED) + list(APPEND JUNCTION_ALL_INCLUDE_DIRS ${TBB_INCLUDE_DIRS}) + # If TBB_USE_TURF_HEAP=1, then TBB has dependencies on junction, so add junction to the linker + # command line again as needed by the GNU linker. + list(APPEND JUNCTION_ALL_LIBRARIES ${TBB_LIBRARIES} junction) +endif() + +# Optional: Locate Tervel and append it to the list of include dirs/libraries. +if(JUNCTION_WITH_TERVEL) + find_package(Tervel REQUIRED) + list(APPEND JUNCTION_ALL_INCLUDE_DIRS ${TERVEL_INCLUDE_DIRS}) + list(APPEND JUNCTION_ALL_LIBRARIES ${TERVEL_LIBRARIES}) +endif() + +# If this is the root listfile, add all samples +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + file(GLOB children samples/*) + foreach(child ${children}) + if(EXISTS "${child}/CMakeLists.txt") + add_subdirectory("${child}") + endif() + endforeach() +endif() + +# Gather source files. +GetFilesWithSourceGroups(GLOB_RECURSE JUNCTION_FILES ${CMAKE_CURRENT_SOURCE_DIR}/junction junction/*.cpp junction/*.h) + +# Configure autogenerated headers. +ConfigureFileIfChanged(cmake/junction_config.h.in "${CMAKE_CURRENT_BINARY_DIR}/include/junction_config.h" JUNCTION_FILES) +if(JUNCTION_USERCONFIG) + # Interpret JUNCTION_USERCONFIG relative to root project, in case Junction was added as a subdirectory + GetAbsoluteRelativeTo(absPath "${CMAKE_SOURCE_DIR}" "${JUNCTION_USERCONFIG}") + ConfigureFileIfChanged("${absPath}" "${CMAKE_CURRENT_BINARY_DIR}/include/junction_userconfig.h" JUNCTION_FILES) +else() + WriteFileIfDifferent("// JUNCTION_USERCONFIG not set when CMake was run. This is a placeholder file.\n" "${CMAKE_CURRENT_BINARY_DIR}/include/junction_userconfig.h" JUNCTION_FILES) +endif() + +# Define library target. +add_library(junction ${JUNCTION_FILES}) + +# Set include dirs for this library (done last, so it's not inherited by subprojects like Tervel, NBDS). +include_directories(${JUNCTION_ALL_INCLUDE_DIRS}) + +# Export include dirs/libraries to parent project if we were invoked using add_subdirectory(). +if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(JUNCTION_INCLUDE_DIRS "${JUNCTION_INCLUDE_DIRS}" PARENT_SCOPE) + set(JUNCTION_ALL_INCLUDE_DIRS "${JUNCTION_ALL_INCLUDE_DIRS}" PARENT_SCOPE) + set(JUNCTION_ALL_LIBRARIES "${JUNCTION_ALL_LIBRARIES}" PARENT_SCOPE) + set(JUNCTION_ALL_DLLS "${JUNCTION_ALL_DLLS}" PARENT_SCOPE) +endif() + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..74a56c2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2016, Jeff Preshing +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0abbc99 --- /dev/null +++ b/README.md @@ -0,0 +1,91 @@ +Junction is a library of concurrent data structures in C++. It contains three hash map implementations: + + junction::ConcurrentMap_Linear + junction::ConcurrentMap_LeapFrog + junction::ConcurrentMap_Grampa + +[CMake](https://cmake.org/) and [Turf](https://github.com/preshing/AcquireRelease) are required. See the blog post [New Concurrent Hash Maps for C++](http://preshing.com/20160201/new-concurrent-hash-maps-for-cpp/) for more information. + +## License + +Junction uses the Simplified BSD License. You can use the source code freely in any project, including commercial applications, as long as you give credit by publishing the contents of the `LICENSE` file in your documentation somewhere. + +## Getting Started + +If you just want to get the code and look around, start by cloning Junction and Turf into adjacent folders, then run CMake on Junction's `CMakeLists.txt`. You'll want to pass different arguments to `cmake` depending on your platform and IDE. + + $ git clone https://github.com/preshing/junction.git + $ git clone https://github.com/preshing/turf.git + $ cd junction + $ mkdir build + $ cd build + $ cmake .. + +On Unix-like environments, `cmake` will generate a Makefile by default. On Windows, it will create a Visual Studio solution. To use a specific version of Visual Studio: + + $ cmake -G "Visual Studio 14 2015" .. + +To generate an Xcode project on OS X: + + $ cmake -G "Xcode" .. + +To generate an Xcode project for iOS: + + $ cmake -G "Xcode" -DCMAKE_TOOLCHAIN_FILE=../../turf/cmake/toolchains/iOS.cmake .. + +The generated build system will contain separate targets for Junction, Turf, and some sample applications. + +[[https://github.com/preshing/junction/docs/vs-solution.png]] + +Alternatively, you can run CMake on a specific sample only: + + $ cd junction/samples/MapCorrectnessTests + $ mkdir build + $ cd build + $ cmake .. + +## Adding Junction to Your Project + +There are several ways to add Junction to your own C++ project. + +1. Add Junction as a build target in an existing CMake-based project. +2. Use CMake to build Junction and Turf, then link the static libraries into your own project. +3. Grab only the source files you need from Junction, copy them to your project and hack them until they build correctly. + +Some developers will prefer approach #3, but I encourage you to try approach #1 or #2 instead. It will be easier to grab future updates that way. There are plenty of files in Junction (and Turf) that you don't really need, but it won't hurt to keep them on your hard drive either. And if you link Junction statically, the linker will exclude the parts that aren't used. + +### Adding to an Existing CMake Project + +If your project is already based on CMake, clone the Junction and Turf source trees somewhere, then call `add_subdirectory` on Junction's root folder from your own CMake script. This will add both Junction and Turf targets to your build system. + +If you use Git, you can add the Junction and Turf repositories as submodules. Otherwise, you can just copy the Junction and Turf source trees to your repository. + +[FIXME: Create a repository with a sample project that demonstrates this.] + +### Building the Libraries Separately + +Generate Junction's build system using the steps described in the *Getting Started* section, then use it to build the libraries you need. Add these to your own build system. Make sure to generate static libraries to avoid linking parts of the library that aren't needed. + +[FIXME: Use CMake's install feature to generate a clean output tree, so users don't have to fiddle with include and library paths too much.] + +## Configuration + +When you first run CMake on Junction, Turf will detect the capabilities of your compiler and write the results to a file in the build tree named `include/turf_config.h`. Similarly, Junction will write `include/junction_config.h` to the build tree. You can modify the contents of those files by setting variables when CMake runs. This can be done by passing additional options to `cmake`, or by using an interactive GUI such as `cmake-gui` or `ccmake`. + +For example, to configure Turf to use the C++11 standard library, you can set the `TURF_PREFER_CPP11` variable on the command line: + + $ cmake -DTURF_PREFER_CPP11=1 .. + +Or, using the CMake GUI: + +[[https://github.com/preshing/junction/docs/cmake-gui.png]] + +Many header files in Turf, and some in Junction, are configurable using preprocessor definitions. For example, `turf/Thread.h` will switch between `turf::Thread` implementations depending on the values of `TURF_IMPL_THREAD_PATH` and `TURF_IMPL_THREAD_TYPE`. If those macros are not defined, they will be set to default values based on information from the environment. You can set them directly by providing your own header file and passing it in the `TURF_USERCONFIG` variable when CMake runs. You can place this file anywhere; CMake will copy it to Turf's build tree right next to `include/turf_config.h`. + + $ cmake -DTURF_USERCONFIG=path/to/custom/turf_userconfig.h.in .. + +The `JUNCTION_USERCONFIG` variable works in a similar way. As an example, take a look at the Python script `junction/samples/MapScalabilityTests/TestAllMaps.py`. This script invokes `cmake` several times, passing a different `junction_userconfig.h.in` file each time. That's how it builds the same test application using different map implementations. + +## Feedback + +If you have any feedback on improving these steps, feel free to [open an issue](https://github.com/preshing/junction/issues) on GitHub, or send a direct message using the [contact form](http://preshing.com/contact/) on my blog. diff --git a/cmake/AddSample.cmake b/cmake/AddSample.cmake new file mode 100644 index 0000000..434cf91 --- /dev/null +++ b/cmake/AddSample.cmake @@ -0,0 +1,37 @@ +#--------------------------------------------------------------------------- +# This script is included from the CMakeLists.txt (listfile) of sample applications. +#--------------------------------------------------------------------------- + +if(NOT DEFINED PROJECT_NAME) + message(FATAL_ERROR "project() should be called before including \"${CMAKE_CURRENT_LIST_FILE}\".") +endif() +if(NOT DEFINED SAMPLE_NAME) + message(FATAL_ERROR "SAMPLE_NAME should be set before including \"${CMAKE_CURRENT_LIST_FILE}\".") +endif() + +# Were we included from the root listfile? +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # Yes, it's the root. + include("${CMAKE_CURRENT_LIST_DIR}/JunctionProjectDefs.cmake") + ApplyTurfBuildSettings() + add_subdirectory(${JUNCTION_ROOT} junction) +elseif(NOT JUNCTION_FOUND) + # No, it was added from a parent listfile (via add_subdirectory). + # The parent is responsible for finding Junction before adding the sample. + # (Or, Junction's listfile is the root, in which case Junction is already found.) + message(FATAL_ERROR "JUNCTION_FOUND should already be set when \"${CMAKE_CURRENT_SOURCE_FILE}\" is not the root listfile.") +endif() + +# Define executable target. +set(MACOSX_BUNDLE_GUI_IDENTIFIER "com.mycompany.\${PRODUCT_NAME:identifier}") +SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +GetFilesWithSourceGroups(GLOB SAMPLE_FILES ${CMAKE_CURRENT_SOURCE_DIR} *) +add_executable(${SAMPLE_NAME} MACOSX_BUNDLE ${SAMPLE_FILES}) +set_target_properties(${SAMPLE_NAME} PROPERTIES XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "iPhone Developer") +set_target_properties(${SAMPLE_NAME} PROPERTIES FOLDER samples) +install(TARGETS ${SAMPLE_NAME} DESTINATION bin) + +# Set include dirs and libraries +include_directories(${JUNCTION_ALL_INCLUDE_DIRS}) +target_link_libraries(${SAMPLE_NAME} ${JUNCTION_ALL_LIBRARIES}) +AddDLLCopyStep(${SAMPLE_NAME} ${JUNCTION_ALL_DLLS}) diff --git a/cmake/JunctionProjectDefs.cmake b/cmake/JunctionProjectDefs.cmake new file mode 100644 index 0000000..cdbe6e8 --- /dev/null +++ b/cmake/JunctionProjectDefs.cmake @@ -0,0 +1,17 @@ +# Add cmake/modules to module search path, so subsequent find_package() commands will work. +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules") + +# FIXME: Implement FindJunction.cmake that other projects can use +# If this script was invoked from FindJunction.cmake, JUNCTION_ROOT should already be set. +if(NOT DEFINED JUNCTION_ROOT) + get_filename_component(JUNCTION_ROOT ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) +endif() + +set(JUNCTION_FOUND TRUE) +set(JUNCTION_INCLUDE_DIRS ${JUNCTION_ROOT}) + +# Find Turf +if(NOT TURF_FOUND) + set(TURF_WITH_EXCEPTIONS FALSE CACHE BOOL "Enable compiler support for C++ exceptions") + find_package(Turf REQUIRED) +endif() diff --git a/cmake/junction_config.h.in b/cmake/junction_config.h.in new file mode 100644 index 0000000..37dcd70 --- /dev/null +++ b/cmake/junction_config.h.in @@ -0,0 +1,11 @@ +#cmakedefine01 JUNCTION_WITH_FOLLY +#cmakedefine01 JUNCTION_WITH_CDS +#cmakedefine01 JUNCTION_WITH_NBDS +#cmakedefine01 JUNCTION_WITH_TBB +#cmakedefine01 JUNCTION_WITH_TERVEL +#cmakedefine01 NBDS_USE_TURF_HEAP +#cmakedefine01 TBB_USE_TURF_HEAP +#cmakedefine01 JUNCTION_TRACK_GRAMPA_STATS +#cmakedefine01 JUNCTION_USE_STRIPING + +#include "junction_userconfig.h" diff --git a/cmake/modules/FindCDS.cmake b/cmake/modules/FindCDS.cmake new file mode 100644 index 0000000..c56cb1a --- /dev/null +++ b/cmake/modules/FindCDS.cmake @@ -0,0 +1,25 @@ +#FIXME: Deal with builds from different versions of Visual Studio +if(NOT CDS_ROOT) + find_path(CDS_ROOT NAMES cds/init.h PATHS + "${CMAKE_CURRENT_SOURCE_DIR}/../libcds" + "${CMAKE_SOURCE_DIR}/../libcds" + "${CMAKE_CURRENT_LIST_DIR}/../../../libcds" + C:/Jeff/libcds-master) # FIXME: Remove this one. +endif() + +find_path(CDS_INCLUDE_DIR cds/init.h ${CDS_ROOT}) +if(WIN32) #FIXME: CygWin + find_library(CDS_LIBRARY libcds-x86-vcv140.lib "${CDS_ROOT}/bin/vc.v140/Win32") + find_file(CDS_DLL libcds-x86-vcv140.dll "${CDS_ROOT}/bin/vc.v140/Win32") +else() + find_library(CDS_LIBRARY cds "${CDS_ROOT}/bin/gcc-x86-linux-0" "${CDS_ROOT}/bin/gcc-amd64-linux-0") +endif() + +if(CDS_LIBRARY AND CDS_INCLUDE_DIR) + set(CDS_FOUND TRUE) +else() + message("Can't find CDS!") + if(CDS_FIND_REQUIRED) + message(FATAL_ERROR "Missing required package CDS") + endif() +endif() diff --git a/cmake/modules/FindFolly.cmake b/cmake/modules/FindFolly.cmake new file mode 100644 index 0000000..7d4b3f7 --- /dev/null +++ b/cmake/modules/FindFolly.cmake @@ -0,0 +1,9 @@ +include(FindPackageHandleStandardArgs) + +find_library(FOLLY_LIBRARY folly) +find_library(GLOG_LIBRARY glog) +find_path(FOLLY_INCLUDE_DIR "folly/String.h") + +set(FOLLY_LIBRARIES ${FOLLY_LIBRARY} ${GLOG_LIBRARY}) + +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Folly REQUIRED_ARGS FOLLY_INCLUDE_DIR FOLLY_LIBRARIES) diff --git a/cmake/modules/FindNBDS.cmake b/cmake/modules/FindNBDS.cmake new file mode 100644 index 0000000..a79d46a --- /dev/null +++ b/cmake/modules/FindNBDS.cmake @@ -0,0 +1,37 @@ +include("${TURF_ROOT}/cmake/Macros.cmake") + +find_path(NBDS_ROOT NAMES + include/hashtable.h + include/map.h + include/rcu.h + include/common.h + PATHS + ~/nbds) +if(NBDS_ROOT) + set(NBDS_FOUND TRUE) + set(NBDS_INCLUDE_DIRS ${NBDS_ROOT}/include) + GetFilesWithSourceGroups(GLOB_RECURSE NBDS_FILES ${NBDS_ROOT} ${NBDS_ROOT}/include/* + ${NBDS_ROOT}/runtime/runtime.c + ${NBDS_ROOT}/runtime/rcu.c + ${NBDS_ROOT}/runtime/lwt.c + ${NBDS_ROOT}/runtime/random.c + ${NBDS_ROOT}/datatype/nstring.c + ${NBDS_ROOT}/runtime/hazard.c + ${NBDS_ROOT}/map/map.c + ${NBDS_ROOT}/map/list.c + ${NBDS_ROOT}/map/skiplist.c + ${NBDS_ROOT}/map/hashtable.c) + if(NBDS_USE_TURF_HEAP) + add_definitions(-DUSE_SYSTEM_MALLOC=1) + else() + GetFilesWithSourceGroups(GLOB NBDS_FILES ${NBDS_ROOT} ${NBDS_ROOT}/runtime/mem.c) + endif() + # FIXME: This is hacky because it relies on inheriting the same include_directories() as Junction + add_library(NBDS ${NBDS_FILES}) + set(NBDS_LIBRARIES NBDS) +else() + message("Can't find NBDS!") + if(NBDS_FIND_REQUIRED) + message(FATAL_ERROR "Missing required package NBDS") + endif() +endif() diff --git a/cmake/modules/FindTBB.cmake b/cmake/modules/FindTBB.cmake new file mode 100644 index 0000000..819e9d8 --- /dev/null +++ b/cmake/modules/FindTBB.cmake @@ -0,0 +1,221 @@ +# The MIT License (MIT) +# +# Copyright (c) 2015 Justus Calvin +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# +# FindTBB +# ------- +# +# Find TBB include directories and libraries. +# +# Usage: +# +# find_package(TBB [major[.minor]] [EXACT] +# [QUIET] [REQUIRED] +# [[COMPONENTS] [components...]] +# [OPTIONAL_COMPONENTS components...]) +# +# where the allowed components are tbbmalloc and tbb_preview. Users may modify +# the behavior of this module with the following variables: +# +# * TBB_ROOT_DIR - The base directory the of TBB installation. +# * TBB_INCLUDE_DIR - The directory that contains the TBB headers files. +# * TBB_LIBRARY - The directory that contains the TBB library files. +# * TBB__LIBRARY - The path of the TBB the corresponding TBB library. +# These libraries, if specified, override the +# corresponding library search results, where +# may be tbb, tbb_debug, tbbmalloc, tbbmalloc_debug, +# tbb_preview, or tbb_preview_debug. +# * TBB_USE_DEBUG_BUILD - The debug version of tbb libraries, if present, will +# be used instead of the release version. +# +# Users may modify the behavior of this module with the following environment +# variables: +# +# * TBB_INSTALL_DIR +# * TBBROOT +# * LIBRARY_PATH +# +# This module will set the following variables: +# +# * TBB_FOUND - Set to false, or undefined, if we haven’t found, or +# don’t want to use TBB. +# * TBB__FOUND - If False, optional part of TBB sytem is +# not available. +# * TBB_VERSION - The full version string +# * TBB_VERSION_MAJOR - The major version +# * TBB_VERSION_MINOR - The minor version +# * TBB_INTERFACE_VERSION - The interface version number defined in +# tbb/tbb_stddef.h. +# * TBB__LIBRARY_RELEASE - The path of the TBB release version of +# , where may be tbb, tbb_debug, +# tbbmalloc, tbbmalloc_debug, tbb_preview, or +# tbb_preview_debug. +# * TBB__LIBRARY_DEGUG - The path of the TBB release version of +# , where may be tbb, tbb_debug, +# tbbmalloc, tbbmalloc_debug, tbb_preview, or +# tbb_preview_debug. +# +# The following varibles should be used to build and link with TBB: +# +# * TBB_INCLUDE_DIRS - The include directory for TBB. +# * TBB_LIBRARIES - The libraries to link against to use TBB. +# * TBB_DEFINITIONS - Definitions to use when compiling code that uses TBB. + +include(FindPackageHandleStandardArgs) + +if(NOT TBB_FOUND) + + ################################## + # Check the build type + ################################## + + if(NOT DEFINED TBB_USE_DEBUG_BUILD) + if(CMAKE_BUILD_TYPE MATCHES "[Debug|DEBUG|debug|RelWithDebInfo|RELWITHDEBINFO|relwithdebinfo]") + set(TBB_USE_DEBUG_BUILD FALSE) + else() + set(TBB_USE_DEBUG_BUILD FALSE) + endif() + endif() + + ################################## + # Set the TBB search directories + ################################## + + # Define search paths based on user input and environment variables + set(TBB_SEARCH_DIR ${TBB_ROOT_DIR} $ENV{TBB_INSTALL_DIR} $ENV{TBBROOT}) + + # Define the search directories based on the current platform + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(TBB_DEFAULT_SEARCH_DIR "C:/Program Files/Intel/TBB" + "C:/Program Files (x86)/Intel/TBB") + # TODO: Set the proper suffix paths based on compiler introspection. + + elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # OS X + set(TBB_DEFAULT_SEARCH_DIR "/opt/intel/tbb") + + # TODO: Check to see which C++ library is being used by the compiler. + if(NOT ${CMAKE_SYSTEM_VERSION} VERSION_LESS 13.0) + # The default C++ library on OS X 10.9 and later is libc++ + set(TBB_LIB_PATH_SUFFIX "lib/libc++") + else() + set(TBB_LIB_PATH_SUFFIX "lib") + endif() + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Linux + set(TBB_DEFAULT_SEARCH_DIR "/opt/intel/tbb") + + # TODO: Check compiler version to see the suffix should be /gcc4.1 or + # /gcc4.1. For now, assume that the compiler is more recent than + # gcc 4.4.x or later. + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + set(TBB_LIB_PATH_SUFFIX "lib/intel64/gcc4.4") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$") + set(TBB_LIB_PATH_SUFFIX "lib/ia32/gcc4.4") + endif() + endif() + + ################################## + # Find the TBB include dir + ################################## + + find_path(TBB_INCLUDE_DIRS tbb/tbb.h + HINTS ${TBB_INCLUDE_DIR} ${TBB_SEARCH_DIR} + PATHS ${TBB_DEFAULT_SEARCH_DIR} + PATH_SUFFIXES include) + if(TBB_INCLUDE_DIR) + set(TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR}) # Set it directly!! + endif() + + ################################## + # Find TBB components + ################################## + + # Find each component + foreach(_comp tbb_preview tbbmalloc tbb) + # Search for the libraries + find_library(TBB_${_comp}_LIBRARY_RELEASE ${_comp} + HINTS ${TBB_LIBRARY} ${TBB_SEARCH_DIR} + PATHS ${TBB_DEFAULT_SEARCH_DIR} + PATH_SUFFIXES "${TBB_LIB_PATH_SUFFIX}") + + find_library(TBB_${_comp}_LIBRARY_DEBUG ${_comp}_debug + HINTS ${TBB_LIBRARY} ${TBB_SEARCH_DIR} + PATHS ${TBB_DEFAULT_SEARCH_DIR} ENV LIBRARY_PATH + PATH_SUFFIXES "${TBB_LIB_PATH_SUFFIX}") + + # Set the library to be used for the component + if(NOT TBB_${_comp}_LIBRARY) + if(TBB_USE_DEBUG_BUILD AND TBB_${_comp}_LIBRARY_DEBUG) + set(TBB_${_comp}_LIBRARY "${TBB_${_comp}_LIBRARY_DEBUG}") + elseif(TBB_${_comp}_LIBRARY_RELEASE) + set(TBB_${_comp}_LIBRARY "${TBB_${_comp}_LIBRARY_RELEASE}") + elseif(TBB_${_comp}_LIBRARY_DEBUG) + set(TBB_${_comp}_LIBRARY "${TBB_${_comp}_LIBRARY_DEBUG}") + endif() + endif() + + # Set the TBB library list and component found variables + if(TBB_${_comp}_LIBRARY) + list(APPEND TBB_LIBRARIES "${TBB_${_comp}_LIBRARY}") + set(TBB_${_comp}_FOUND TRUE) + else() + set(TBB_${_comp}_FOUND FALSE) + endif() + + mark_as_advanced(TBB_${_comp}_LIBRARY_RELEASE) + mark_as_advanced(TBB_${_comp}_LIBRARY_DEBUG) + mark_as_advanced(TBB_${_comp}_LIBRARY) + + endforeach() + + ################################## + # Set compile flags + ################################## + + if(TBB_tbb_LIBRARY MATCHES "debug") + set(TBB_DEFINITIONS "-DTBB_USE_DEBUG=1") + endif() + + ################################## + # Set version strings + ################################## + + if(TBB_INCLUDE_DIRS) + file(READ "${TBB_INCLUDE_DIRS}/tbb/tbb_stddef.h" _tbb_version_file) + string(REGEX REPLACE ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1" + TBB_VERSION_MAJOR "${_tbb_version_file}") + string(REGEX REPLACE ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1" + TBB_VERSION_MINOR "${_tbb_version_file}") + string(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" + TBB_INTERFACE_VERSION "${_tbb_version_file}") + set(TBB_VERSION "${TBB_VERSION_MAJOR}.${TBB_VERSION_MINOR}") + endif() + + find_package_handle_standard_args(TBB + REQUIRED_VARS TBB_INCLUDE_DIRS TBB_LIBRARIES + HANDLE_COMPONENTS + VERSION_VAR TBB_VERSION) + + mark_as_advanced(TBB_INCLUDE_DIRS TBB_LIBRARIES) + +endif() diff --git a/cmake/modules/FindTervel.cmake b/cmake/modules/FindTervel.cmake new file mode 100644 index 0000000..8aaa404 --- /dev/null +++ b/cmake/modules/FindTervel.cmake @@ -0,0 +1,17 @@ +include("${TURF_ROOT}/cmake/Macros.cmake") + +find_path(TERVEL_ROOT NAMES tervel/containers/wf/hash-map/wf_hash_map.h PATHS + ~/tervel-library) +if(TERVEL_ROOT) + set(TERVEL_FOUND TRUE) + set(TERVEL_INCLUDE_DIRS ${TERVEL_ROOT}) + GetFilesWithSourceGroups(GLOB_RECURSE TERVEL_FILES ${TERVEL_ROOT} ${TERVEL_ROOT}/tervel/containers/* ${TERVEL_ROOT}/tervel/util/*) + # FIXME: This is hacky because it relies on inheriting the same include_directories() as Junction + add_library(Tervel ${TERVEL_FILES}) + set(TERVEL_LIBRARIES Tervel) +else() + message("Can't find Tervel!") + if(Tervel_FIND_REQUIRED) + message(FATAL_ERROR "Missing required package Tervel") + endif() +endif() diff --git a/cmake/modules/FindTurf.cmake b/cmake/modules/FindTurf.cmake new file mode 100644 index 0000000..2f97d4d --- /dev/null +++ b/cmake/modules/FindTurf.cmake @@ -0,0 +1,27 @@ +#---------------------------------------------- +# When Turf is found, it's basically as if TurfProjectDefs.cmake was included. +# +# Later, this might find binary packages too. +# +# Sets: +# TURF_FOUND +# TURF_ROOT - the root folder +# TURF_INCLUDE_DIRS - pass to include_directories() +# AddTurfTarget() - call this to actually add the target +# +# You'll want to set the compiler options before calling AddTurfTarget(). +#---------------------------------------------- + +find_path(TURF_ROOT NAMES "CMakeLists.txt" "cmake/TurfProjectDefs.cmake" PATHS + "${CMAKE_CURRENT_SOURCE_DIR}/../turf" + "${CMAKE_SOURCE_DIR}/../turf" + "${CMAKE_CURRENT_LIST_DIR}/../../../turf") + +if(TURF_ROOT) + include("${TURF_ROOT}/cmake/TurfProjectDefs.cmake") +else() + message("Can't find Turf!") + if(Turf_FIND_REQUIRED) + message(FATAL_ERROR "Missing required package Turf") + endif() +endif() diff --git a/docs/cmake-gui.png b/docs/cmake-gui.png new file mode 100644 index 0000000000000000000000000000000000000000..ebf0403617f738ad2e55f69ca48bb68efa5013b9 GIT binary patch literal 24311 zcmeFY^;cd&&nUWacbDSs6n87d-QC^Y-5rWcfg+{2ySqb?BE{X^{XBi&@0@$r`QiQr z_gQPN*^@oVB$-SmGnpq!QCb603<2NFUkM_&H#GyVIe?}gp~zPPyy#CspSFy z2pIn!Fd!=j7XaYpti;3=6)hcH9b7CO97&|a#7G>S9n7t4%>cmj?{^gs&EyAc!Howq z6m`+1%i46?Ab^a#0(Jyds{r*IU=)gqE<6|(39pWcg_a|!HW2&=Uh;e7c0Nsug-CRy zc8r=X_c`%uNu!_l#o5`*;!DRt&cW-H*EC8X8ah?det|W>qa4Q1yF{FzAUQDF83>Lg zxd};vx<065_Ihlv7xc$o;sN>Gs}kAlKLCpO+wpP>#hZfZwZkH@4J)0au z@&H113K0PCpJsaBxrGuB_IJ1T%@1rZZU+nAZ7v5IPxdwUBI zj>BLzZrwJeI>29bz;feBk8K4mUmO0n_fw1)_p2~b?hnNg{!fJP2nc7C#y>h)c47oS z*tQ%tWBl(yU*A`}F9D}4;mpdO!jLy7BYXEM<@kefNKsb%hXeBW;jnFNFq5Z}g`D zPh;NNM9r@iROh5bq`8#()W{eH$&wVvqu+Mr;YleIKT9D<*+{NPG8U;63C^Ng<;YT< zB-)Lx@50svR*y1rV;VdgJRiIg+-5xHOxd4MbLp8Qsw|722{J7*P%<3UH0kc? ze6fab56V>4T-1dAc$FcjmaCzZ>#O_y{Hj*3xKj!@k6r$!^pje9DQDG` zvWIel;l+yf{+qq0eSlnxYHN>JTRkHW_lpI(1K{ zvhGkX1n$f}vMOROW#$#;N#@`8ohHGMhiW%3d6sjer%0!erhFJ77=;*X7pyDIEB!0E z{(ArI`fGgVe1?5?^7q2c@w@PMhVOU0cDxU6aBZAzN^N0nD{j-rf^#M(UPo|uV|U0$ zYcmpecDIFhIL8+Y7c<0MEbOqHiFmfG+$NV3i6)0O8CESbm!Vs~(8ev>k_IdbXQT?L zTd2+>EOt0NaRkjP90-jo*w@WgZTO9htv;_54HOOTcF|6z*kqZv%{0upPE3q2tX3>l zjGo5x{K>J&HR@XKSb2>LD;Ccgl#bzy8BY6~<~t6RCYSb5k)bZA?xlXGzE~l(pnJk_ zVz;1St4N@cjhM}wJ>K-IGP`oXJ=<;ra~m^(c9!-|#r`kr>FX){Y1S$98PC1ZJ>|XK z{q&~^LiA4ye6D=H&aQ3~?zWB{Cq-NA{iOq?{YM%W1QtJBzTa@*{FF%?M^AT8$5F8> zi7v6v^Bzy#!#{7EZJfI+%x=$Ktz5R7Uz-pZ^ecanW6CBN)$13y5V^g4(A;*K*!f

A(tQTw#tTPBds2;)wf*3LrvKWQ~Dh{e0 zni)|LOB4|fNsHJy*`WvLn&8QVZPkJyYX)l!_B)m~!86MQl`PQ$T7394>;|g8!_Oz# zqlqTi$V~~`$QEKNyiA-YKTKM>&t*)r1`(? ze^*bv=U(IU6MMD{SNQ5S#0ICJOz)M1_pO!uMo#_9!g|H@vePmnPfAWoPN6cARayi6 zB1}-r4~=Et4*?Q@pf$T7!@23CRRP=Hd9Zl%UQp1%kjFj zH^psDWwmQ-uhr{#<&fFpVlnz)4ffR1l+khfs$1<=6D*@tQ?(20^glZ40dzR21{IDK zzp5T2=p1-qv^VlMRz&lMoz69A`QcO%*HGKR65Ty}t+OjjO*^mN$1TQ~h1&Ui0v|K@RzZ z#kYQua@x^I@2O$aDeiY?KYJd$mVj?e(AK^0zt$q*p)i=6;8W}ab1!~zV(sc+yIS4d zT?c2U%o#@{$?Le69q8~k`z*_&(NUWYxld%TmwMx;ug{L+;g6~Q)&9wi%z)-Q*b_-F z`dQv?w+hBWyL^Ekofn6(h2KBvf0F2G{_wwG-7#?+=19=U&Hqj#aP_nsoV-(FT9W%+ zUPxd*bJgoX_BGQ+Pf4%SLTox$2*>qYw%Ooe^k)4aLGQA|@s{ktc!jcArc$6vK*Hbg zZA-FR$Kk~6Vf;*kzsmJi?u2jVVlLp>y62C};^QKdUY=Ly#n351vpv-sVGCv3-J|>c z8IA}1x$LT*cYR-uFc*8B`}RkyMrg0B=eF2n+#$rw>qh1ORR<0B~#w0K6Fh zfaCDZVDKvdKmdxes^TD=kB^Tte9{*4mHBB3hN zvycPO;RZ~NxuQ7PwK&)VL3jYr4FIM9z{SZK!ONS-&20(}t#0d75fk6a#2ImU{a_U} z!)sWP*z_J!v<+}Y6^=aLKYv_5yqeTLaYxVS74HDXRREhN6bx+3#%fJ*=f001GN?7aN>Fvi#!%MEEs7&+O=+;N*Dt?e*bkDDYz?@p3BS7?;Cr3v|`}_NQ zdwV-OJDZ!E8yg#IYilbjD@#jDi;IhY{`{GlnVFuRo|u>zA0Hnc9v&DN=iPGy(??TPBbW2}ynu-$KM>Oy zO`Js7s?c?IR^ZD~1L@F_E8nPWHe#;~4A!uExV-$n*Hq?@7e0j&^W3BGi_B?MX40=Z zaDSENq++Dl?dGYP*z&&bs!R|TuoiV}L)GpYD~T(eyw+a01%a2pYV7mEgRQ_k|C?@mD z0-Y^YS<*3C+bl-_f}2I&7~uiIthsNdBP>6wn#Js(K+T=rc~Mk0vnTNx6ChM^Wzx>1 z&oSQa19LBHgo6+H-RnqZ!5ze?jitf;z7#L%sHAFx;Xzo}PckQN9HZWfy3OrK$o*vo z-&@orte*nfJR+AkkQ=(yBIyNFf?_yGC005#il^7ikO7_(Vxtu{HV7M?huf0MX_K1_ zAt{WNWk1>&$Y4QqLevgR`dT>S2x2DV7Xm2@%Mk8?G1HzMw&h;@Agh!~;W3$HWKc{9 zH>y*tiMAIb=AhLjeHmGBpuyKbr$9aC%3Ka*n1P_e+K7C@WbXK#7D=!7vZV7f7s8~l z{Lt_@N5BHdtJf-&2Vt75j3l_u0&LhRvLS}vJdPWoX%0Pd(+*d8LV5GvJV?b znr)$Y-zJ2^4~IS1-GU>J6%(EAyQ;@Usk2@As>V4JK?)>^ausZ)+~9ds*c8%bANs|} zBFJadFOi2qHNc}q$scR?A<70DEL30RRkL1{lWzTt))VNx^UkKs6xrThF1 z4yakrn5IrYzFwQ*jZnX?E6`to^gQYxs^N-zuH9vnv z3#=QP698umg-B2Y6gj0{d0ZmSR!J%~Ii=eB-KSPEmGuNWa=t-~UzJ`c8Sgw-R)baB z>2=Q#sI6i&@CT4}ibs1GmDkzO#t3?XEd_P=MIZxkd78K^zT! z7FexquyZa&cSa@SYJWd97{qoHpkqoPfR#8>iho}FuVbjsIWkaTF{~Xi!X-^aBJDv{ zzE^%i)N;luLt!KHtc+UNClvmUw%Jc2l_pIBNu)0CQhgFeNp{ zQw}7+b`Kk`bDFdAwGt1GO@XwG6;Hj~R2^q+)vH?P%5GSi58b@XyPTJi_ri_9 zT~3X~(TB{vq5+}A6Q);7T-D9oh|LUXOm1$msmh@Hu5soYWv)}q?~j54o7(E*L}U%B zj&#KQj;)zJAt<`MdEi$-%1$Ve+0BOchE!c)+6i|tvKpt9(5+e8(ObQWg2UZ6wq7m` zPW#(*M*Iyn?DO`{3x7oM#o0sNJctwvz_R1RUY9NK*O$sIj{I|>Xcr?Lmx$(a8qp)< zkcNt@rYaqvY-H2O zIZTVVs`GeF+HO0G>te(OGhAg|fFBBcR20}OcX-p9P@4mR9L98jX5RB$z#AdpjcuV{ z|DKU|#v8Mgt0h*Uw5JFS5!iWubJ|@2)HHVtQm{pTZvuC7S& z>0BtTkixI8k>j=tQvDfYHt-iVph!iWJA`|tI@UHm0%{Qw{!pV3* z9IVkai?YvwaFYqK!OM}NchruOhwpCK7kN3p3%1w;xUi;63RR-WlyqDlwxUjCWBHJB zmNF^v73qnVxjzQMfdyB$9jt5k@?hSM)5v&YUF;(MY1@Nq{}u)ILn4j!D>=8TE23E= zM2nmpcJzZ5EF>91(1<8W9cm=Q|Gf6>2#(XMl0$;M&DWz^;*z0i!G6K+^UT=1wv;P{ zy2d{BI@E!=PV>s?Y(oOB)5`ff@Cy9u+rOJli;Y8flL&spoc1vkJxqjhyGa9kwD9!% z8ql^pjcvOjcFV+n`9@ibj%9(IyeUHYCO@e`;Fz#b<4TdK<=gvz;MMev`U#< z!g$3oqDSfv;81su5ljB?*n@Da;+yr}j9BLIN+?>Iz7Kd2UjMvac-ON-OybZQkxmC; zaxNR_VIXgZRbX0LArP~A5t+i;9B^u<@ABJrx&A|^4+hn5{?r`36e_SwP_@g)W0D90 z(Q91WYk&?7UT7g_&{iS&7uR)C?RCo~ zSkL>sj(xzJpF9v5`$bP5PMt`DuYd_ohF$3S=5YDP$_U-y~KY2tC zZ2nD-!#RC3%0BE9wOC zcvEu(ht?rAESwEgYK#VvtXYKEw5S6r_HX(A-@i4*_~D!*L@{*1lfpoVp+Z^=ZW3=% zJ0)6%;3*39_wVAo>RPD5vT1FFHLolF4euvlKzNIJ5L%a{QRMi& zvpH%+O~{_Uo10sRfw=>e56AposQ*njxpm$21A@6~yxCesz{`xvW47n}q^kFOf%~R; z6N*cxKa9OM-xfI!V(m*Uy3E|nx868yl-*1n1xUK_$l;=NA8U4eozM)ymsdi$@w?Wh z`%?sj0GHFh{0J8B+s>|Hf91>5*aDoqkPWR4<6rM;WjsDRvF>L;t<^0P0*e8Dr?a1s zECen7aJpJ(;~THbuUMTeW+v!mlVvr)Z%k>`Gv++q?9$>-y$ z)E`5DYaA5wxZ1s@F$qS~v*pWf0mmmswogNCNuKEwWFNQcKo*5h4M zygfD7EnXovvO57;>FbAjULDrv|9Rzx2WaCECZ>6VPG9DTd^C;RF>BbFScLd^&hx#r zN>Cm5uF6hJ<;-t9z9h$}WKZm0418i|KCzACv(%qUrjfWORM$^P?WUN>!t+SfSwmPo zl6!w|)*w9KSQQZ!3)Drv}omLWn%i4Ra=KjSjC^ z?%9SHGj<7<44oRjq`n#6*KtzMZxtiUtITL z4}G*&^3H+v{ZLYT0MV)hv3*BPD)4QYpujC)83}iGdk8{nTlo(4#4&2;WfaB1mqEGA ztLsw}gi>;=2!5Zki{|r(qz^Xt#8J&h!1n4-a(gR|I{03Ca}xSaNa2q_V(cUl5<`W#1%f9aZ37e;WLG^BL*qbPo zUVVLD{WhL#2tDy`E4D!3iJ8u2x_-s4-lU_-*I8{za9}mcKs#jh(@E2^qtDoE)*H{( zbV@1*b!_X5Xbi54(Mtk9_XmewnU`sCdQWQuz7#D~2>X|NmO4U&(KsctOqGwsmh(qY ze#x=oPx^4%4o=Nh|IyC5cxCl#@CWS`j)?Di(lSsj4ysC?#@0A1GWya zlXcy~Kdnu_yCNPrA+7f>oIIK&*IcJCvbKPJa3Z@1yfb+*E)P&@u>W> z*v09NDtZ4%%JN_z@z>*c*@6dff0{2|O&U*3+I%*POjT`D-f%Iy)x6BPj z9#g}*9=S$a^Ki(STd{ubd@3(8Ka~7uMNd@cD@Rd}(-}+Pk|#Is?)T<4=)F%A;;$Vqb-@ya9j`V!q9dEzI2B>#WXszNoV7L#_pUZGjh;_29m#*XGA?j2c ztz8|UNKi{*P3b&Y z<^YId@3yhTR2_UjPrXKLt(3DpPn%KJryd=b#2kojD-7Cq{-_7(^S;!xc1_csj$)ff zM^`SsEwuL~9sKm_Rp*b5!(oKN7$^7{kJGkxKMZ4I(P3yM8Iv6Q!gnEUZxr~$V;DjG zJdQ1g@haqj!QVD^nr3aG*3hbv6$emX$QhANZ`SvySoTd5x?W|UUWtBc5UM{LcqKAp z+SqS)Z`V^ubRQzv&Z^g#-yFI9bRxd9f>kYL{{y+(efUP*Wm|y2A_KNR?h0%Tk9h!I zuA|^DSl&dV&}Q6~(c_BZQB?%9;TmVhcecnua$2O(N&g{uy$&?(bc}FQxZSD8PHwC| zN;$A3vzCk+f}NEphJGl!Sn9e*p~?RC#1zKR{!gr3hdP91$$^Oo>3z^v zqw|Ji;&?Pcs9Fd4h&=Q6$`OO9#!M1fUb9-O63%%V{de5S8Axy* zg8?+xg!(-*p3x>Z5O9OYQQ1l_cDm0EqwqB-+C;M$$m!NIwx6*t9W)6(G0FQ230zX% zVgErlfQZb7iKsI0b0wA*96(2UK^3Z70n_zzVHNx=a3FE7v960adOP-lB0vONr@3td zuC#>`N-se~-}<;PqtsmQ{!xYk!D4`iitNuRIlpx_=-eX6G{cVvc2DZ>X(=eQhyQ3f zdGSi;eBC1nR8BDKr~R~OB7LZ)+UQtzJInOg zpA@VsxC3n<9E==3?jjVN5s9}Wh$nEkPF}~=4taexbYGK758uQ;oedi3JszcU;p9!` zbe3$6##~@d_`B3D<-N*KRGq%GS34Rh&Fkj0U3$RmXRLo54SIaeC1BNJuVu+JMB^$$ z$lkX2Ilmpz$LVo5boX0Cm9RU&Zm+j&p(SMhglrGw9hxur-i9S6(tfhx!R+ZJ3J%9m zh<*8(X-Ek%mr?OKavGS_250-5V6rFT^0GBu!j(Dm_t1mgK?!5$%P?r|N>ppP&ToCI zq4de3eYpxzvrG`yO-rEbdRruXL6xoo!fCUnKpueE*Rt~}AX?BFyHVGdzW@D{ct5>+ z!-{2URmCu1t3QDl#(v_BkTGH1Gf!v!T1|dDr(*wsTJFeYjUBfnrgQRgw9Z=R;KuUp zjocg*KBNDLFJqFo|J?ui0Mz;A%qi-*F33_i$`!o@nrb{)If)jkBI7uyjZBfKgq~OR zs80T5*simaf%AUaz_{hiG`wIPxtJlM_}i1O_Y%T11c8?dNS8Nu?h`V&0bU^ z?ke^Wrwfe2mooP_UF{7jOH3I`S9^v zyR7lj1XrUUhbH@pd7oUYma&~p#6RJFbH+8W-$&JJ`f4l-L()5iD~U6LkB1l>p zh}i#f5g!n0qjy(1nqFAbd)q&-MBC2yyUWhYlarThxt@J6{#ogN8io?C<&_MJRHmJ6bXaI1)L%O&p6bEXF%jNJvjI&Yya1cy{@I{zd53@ZGKu z`mi2CxQz-cBTEbQyR}fbRF*3Fhe6q!w-W#C4)sxDE(k{bOg5)-HXOLrUMjg*)c4ps z7|?|r=wlHt)V}JML^+DsT=t#ZpLu&6lguTpkrZo4XD)u;N%-WXOA1Hmb(Qn$t=3Zt zJfI1ux(OqUtw{R%0C800$l&`fBzjFpf*@>adR(NGNNIaH=J@_og4lcs_sJo0PUDgP z&tA?nu!=m767`jd%9$+D%y_%&*yL7|ql~YmAFOl6FPH={{SkMPD~ViQVWMtj3uoisGg#pFf9=SPjQbX6GuJx=L|B)Q`SKpL4kuSI(9g(^w1)m< zH#b+4Vn1rB;NocwP20ViK1=m|WEH?M1*!kPsQ`iMCqzJH^(iq7W#=-_6m!>H2x1T78YSSq7I`GdjVc~Y3pux+{*UY`Cam1#RRn`0hi#5fo7xon@Sk} z487!+N90@0tNT>R=OG>cc-XsoMCqW%%MjDBzxts0*^~Au>?D4Mk zgt4@4Tke2~j;*P!h46%lY4=bSsrv7P(9i?lM&{~j=p_&D8CWWLkb$%8B*^)_)CPRS z8h6k^g2|-T1af%+@40{OC)%_xqPW-;uA{;0-@1XcH-x(6c$(xBoKT&`Gu^vCOvx5f zebppC_z+27-h*jbD6AW+X;tiO`m0W+&b_$0zA#1E>k)7N3^frE&0rWgg!0bgy~_Le zh5x-msH$omL*L9z_A`z2Hvz}4=Q$rEtwb28^MJseA5oFN*OOFvzqNAg9H50csiqk& zs=hE6l^rV(IU*sl;JC07xwhw_{x*sf{?AEUsG(8f3&V~T7wnlW9Fta4LZPyKNjNf- zmgUR9mjQZq3)0_^>8Q%4wB@D@^)0uQ?{`=yaZkWPCXjM`t)y;6=NHAAR`?m3lV!*m zY`t7H-QXGBgrS+e4)))<^u)HZUY9Q!jPZn9tW@|pE3gclID^@;q;?Htuo8==AjMK* zKQX}tpv`iT7HE_cE?erw8@BhV5~Wt(>mCf3i@%~A-aSi`7o*!5W)A+4OS?`XCwy2k zn}1Ug44kC_Xg>PqMBN5`=cC(N2|YyYI*c}S?Vq-F^X;n?W!zkN&iR3q3xPhTY2M;T zH<#A9@8ZT2i>(KDE?#2OxPi96H}pg6q_pQJ_xxvdCJe;9`U_WQPeM|P?U^24Rm7UA zJ;k?En7|{;oB^gqAgHqDwN}i%oWo*Ao@S1*b-$NkrG72lu zC^}UG*I>hUJ=-ah>YWhZXC(%B=`?QkJUR#r5Gs~E=~%3cq{V-Z2Lt$|FG$XR^8hva zX!_KcslNu1$2Y^)Y&zB1T~l4xPtxM;z7MkGoQi41uCGnJWl4N;ygVsq3w>lUT}q`P zbkm~~T`qzI%vinK8*znu(F!$8-sifi2p*r0vIVzCB_$KmP0EluhXCMKCvT=jzaeHSfC6CShF*u6`^;x;o{t9Wdce~^Co!?S0QF@Ta1I$7J6qvB8F*NNEP{fa3q zbzNO$5aDaA0BCQ1Mt=dG+TJ(`(CHLq23t-9+%JQa|HS#9LOf9=qyCt^M6B=`y-iWSO;5wUr{y zT8ByTV_EL6AyduDrPT!n-6TUuVS=Q@zo(8Zy4M^MTpsHu*wxGTs!Wdo13j;o#y)g7 z{RoFA&&Lhh+}Mo*XxHKq_*$psMI|Gr^cd?h(2DqXlqNd%IA8|p%{X3c`$9<7!{C99 zy9AU_w!Ysj{gFxN=&*a@>-eHGjy3g7U_;{9^Udji)5AF2f`;%#6QvKD;YM)b%=Pj~ zP@}{++wWCiOVnNyECV*{O6`$*pd|i)vnesKnH-Mx`{D%QkI}CgH$fPhY2df#$6AXy z0C(Y+x09Cc4lH$?eyzpX`Kn=oCcOjN0t=g51al8U+9OR@4^^!jAI{2y1vWAops7Uf zv4u0g@)MhBFQSSm>R!{XWbpxtM}eTs*go;8WMoL?DfD&`e}&ysFdFbu4})0qW@u6I zqAJt2Un#T`2zpG3iF5*nkf`$)Ph3t87G3jw9gKe=i)Cvto1G3I0ySS{p@5PqJWJQF zK+gho6)(ur+93+A_7C1C?g@FB-y6&6E?b4_1u_qQ-^}MN5yycCd8e`t#`+CkelbHZ zY30f@boeA7)beLWNWVw_Bms}@hdq-I+mmYH2|u6Rp#F>3y@5LH%}l3_clFg*0RMaK z)zXR$KNR3It~1bB={ZS&yCUQUVe(RPp6Yv_)|~NpYXo`g?ROqiTwGzbUkW3UCPGx6 zqv=6Cj^a!I@fg-Kq>FPH*gH($>t9iKOUe1THs?4Znx0Wd-y`6AR5fsQ2ZI&a9DE7| z;|l2}A?{fJ-z-W;h)Ph`p$bMA1IE#J6+f)a4P0s5C&+Y=Lx*e&5IoYVo!qI;Kud@! zCzUUpSt{DIK?L2VC7jU_thX*e7t93%{%&{_gJmh01NY(Y>=L(8R&Z{4e(nKOz|zBz zy!4|cck#--R=fN0NuJ;RvPCBw7Sne3D`P}x+jNy{@$wVytAw-Ynnm6n)kU1#b0HOS zme3@kuF_N8WwVOw^&xu%4&$w*D|6a3mr6 zEE^sk>puJaJUI;5eAScsdGp0Q%x^kJG&Kev^r=yTffiJ_T{IQj(a}IY2D;~eL-IMfHlWA7pUjLMZJ!})Bl^4PVlgLo z!Om0GQM(yfqPJ;V$uQTnK6GHN1yebFvd2|i2A&krO<*^(ojz_)Fud7}pDh?!A%*)z z1Ch0hd};Luq@#T4hwFjWi~PwCPC36fsU?r~T;-3brY?Qfx0(xnBw&j6F5)j?(%oGi zVCWVVKVp6;*i_YOwvmdR%-8&Gf&gvHL`(iMH3w>Jx8c>;)rIHyWnnRo?LLpkYQqcP zcYnK-C3n+$aE?n|FT~sj5tzKESj^H9{sZ%j2&h>cqcqa;+ZyCS8y;%venjO&Y;GgX zuzgc>`}zL3gD3i{jO{Mi#z9;-Kx)n!lQEadV*R?XAyf=&FI|2G16j?Nndq1GlhV&} z<3AcDN`IW)#ZGFYk@V~~W9YEQ)-Q*nEvmi}SN=jhtmyVQ8j9D7`lN6=HZs5^L7&@# zSXC!BCix|`*jP*>+vHjT3+t)b4GIBRdYTAR*^Q~Ea zr<7i@sB#0s8yfG03YcPTUBfFXW*{*(vQvZPp!PIY1{Aa52e2c(J6F(qzQLOv1@J?# zva#(=kfSN(L7z$c48*(Mu0wH(gSS2FUG@B2iY3x3;{H0O5gwVBSGrMAD?sT16@DFp zmpSz?rSAKIL4i#s3Zb4pVo{WHe>mgPE_S0Mp2uLylmow*?F{G&y%lO|E@oBV`@sbI zU8fs>m)co9?uU7-eD0JLm45GoImZfcP1WmAzyq^PC%%C#=JP!%6@s>(DxirWXHw2= zIgENrBPRDXt@n09JKfC~H+fK#I*rCf#qrF0#+?9JHVS94GmJFU zfq{;dv;cEZ^kaCn@&!?)*#VNov5+5_#)XR}&{wgkrW z`l+C38^?1g0-EmNLr+)?*GZcYNP=H2W68%Bh=E-tDEFUa80~bsp&+VqtL6E~a0|Y% zJ;oZFQ#hTRMMkB2`TQZjxor8)ez^R8UCJ>s1|7H_c?7WJj{~>z$Uu(5tCO?Z8QyNG zy7*uH{Q4s7`a#vv;<9_h>#ma{u%4QGN#+1;{B~rwH6f$Jsu-)|Bk{DwcfzPCWv2h`w|)`R6=s1M&}jxrO@@V7$M$J82IPo9Rw za|@6a0_{#7*3i(}I^E+D(D~gDh1wa|RqOj1%x3l|Fm;dqU$z1g7FR0og)pB9*K%Os zB}5g@qM&j;9B~lSlztfx879y`ZfjVy*J4Y4|Bfe{0Q6m+l7yr>ed!M1#_^m=!bZ$e z(j-GQWp!19Tr)x3%y^|!wy+3f@vZA%y`l>d^LZJ@$VMqD5QhD3L}S~d+6haV|DdH< zHlB`!LLmU5&)91)3BgqgDU}(NZiM<7eI9H6C;Sz(2n!h9S@6HWIcf~vni85g3qA+Z z`YAA398PLlSW)Iu=SG+@l_ZKWFx_ZV4E`A2{=ZlvrFuOlRFdgGl#OwfRf14e>mt3O z>l%V1N!d~Izey{$$b3eWc7sYS3l{AVL6uds#A!LA`lX8WI-^PgnZyvB8&rbXrhTmu93Gu3AbrvW-N>jOC4By$0Ec9%7IMAG(<&KF=0;oq>r`<+VwbO45e#8PPcK zZX)C2M;S>BS0Ik8;xXedH;j2{#CK~t)L?OJk?EfOAJ!}_`$5^N!G}yEERkSMf(~~D zWh7Btu)%P2olvRv!P)<%4kU~J@a#Fkqbb4~7@YtyBmG}yQ2$_Ug6K#8Lpb+FsIw0) zLGS+!Vkht$C8>aZsps#db`p8oN|JXhlv1z!#Q#cL&=#CcYkCEJ<(=Yk_X{N;+Vnt~fXuSyaN zU`UGxIjtkZ!FPP#XD+!mxG^+6h9zH6HAIiZCKp!pC-~o>KdSlb6q=@5GO1#?Pk#S0 zDZ-a1A>}kn1kV@*&vS+jhYBf}=#V_Ykox4LUJkbWt}B8s6&_Tlbx5HKQp!f(7`{0M z{A94&JkO*lqZ#POGoUUdLLGuFR8&$FRh3NKh4qYs~((P_Rp!PDh1wn@L**;<5+js5QamBL8cOd{!oO*+KBp`dt&!34&MG7{9)@F=@9O~ z92#3aYD=<|9A=s+8Ya4_6%_4|!yM4^{zI%zNrbT!7?_YmH$mJ!%0wc~Ij>*piqC;b zB@Dv-0s)&q1IsCm%ZuG@tsPnRm#wsZuFQIR!kIZA z78`UD1`W`pv~J*KWUygja96eO@efg_sq4$2CRy6>k`jIkr`}d;1v(BTTqLp87=Ww* z9W1|j7ku-%^aa~PP`qBQ$>|}55(_xHE>}h*&IMQuPx2Sw=<7f%&U(1%CXbhpK>uGIyLtXCHpDA%z)ztwx`Ls%4TsJO z3hZs#7Fu=?o&Z1<@REXRr}$>5=`-++bl`Fe4w$mwgz#cFgCms{#)9f2fo+QpPG>p- zi>5#oHRjm;sH4RcM~4-m7$;EW!9e{t;kS%vR7L3E3&l{?Fuy41&j_naDvlGw9u+5v z2P=zOnI@4TGgqV5G!mE6KqfFqBkG8Ygyvh4bgnMTy$h?#C#IsOc)?x6_nsKR_L7W8 zs$ueDBJraac^KYE!;N*AdO>YVb0?V<(ZLf%LleQl9bLWeV*$CS8bg5$jUi{!0jk_;hZq9;lZ?#X9{>MP4QBrgAb2>SZ*d6 z-VwGxa!Y%K3D<_Tpkvm>s*8(UV*X>mQ%Q;$9mZ$z*1Id-Qi53(gng<5OJ2dJtVBYC zDoO`Wo+bo~sUs+msZfVVd)L)L#~v+`lvk=jwffsA51NpqI}+ zW?z@Z6#otk0`LL~;T&cCv9z=@R{)ZW_7|ebPB+^ZR@ftNuGH#OX13AD2%yD!d9=2AQ6ASZi95KTt z6waZ@#bsVSqK0^{dw=lA$9V|((7{SvYAxb-jg?g>9M~=Cw5>9lmW5U4ffoCwTHd^A zykEwx{@#;%jzOqR7^_BQG>7e{6G&Da`$a={Xdu2V>0nNa)k82Z(eA~%hd z*$Dk)&8EvKB9CFqG?Sq99fOw8tNMSBs9_qm|1des-t`$Ph#!Pega;u-F%&8*1=>6j zK|o`7?!C-Nnlbo@W<_j*liG&=m;rV7y;h3lb^xgk6Wc`XPp*`-9t|v$VtFVHRZ6s| zUs$tPFgR&x!KGm+_F>mqMrHV`0Vh&r!KH90_Af3tY%S0@kOiw^!jkeMhS7m4uCN8s zeQktWo*|{|%II>x_vmG%4Yogt2Mmt7wE~$Aj+tm#*#en7ah!|GnisV zd9d@U*Rny-`#o8T+^jM}7^PNtWr5)dL->{{@tE;#2#osU_>z)0j8nvv5<(cYR(N@V zp@CO3gZbDbnTFme>bT{y!E&Q!-VCB5L>TSA&%c+`3RZPUl_uYL(x2WXu`Ukf86J(WPw^lGmvX21y;^+ z053&9ell1G;C=UoXjH&WGE=34uMzU;0aj4lky*`@UNHKS84A^uEM13&OdP76B+(_` zJ7$9`nuTF$6QjXJqEMBUaMPq%mOvI<*t=9lgE&qByGGOKkvIZ^get9CkSb1pb`qw8 z@1>9dnHWWc*N}Ek!*3!}4k!?2VJ;;N9sIZcpTVLBK~3z+D!9}xX(YdBei4Rp@g!&U zd6LMWs$!}lv!Yyq3l;rjA);y#QB+DK1i99tsWfycInvWnbVw|yQ6m+RA2%MK_@^c1 zNOdEA;V*ngGTk7Q3c_wlq}3`ZZBh%qLcsUIkgkq=Hm8IC zWosn2d@z6_8u`yOr4yrrWzj%NM3NK_LIvtTQW#-^?iJ40%MvC=i?WQO+hRe*C0bTM z$%b`&3*P_2n$IVgr+|S70r_9egE%oJ>y~o@sqTEONf#*)jLKH82pfKyM@CT7@@)?^Bp6=G) z5C1tA6srF+>$AnhC&z*^B1?YI1HwTYTs*&Ivf>8NSdEYig#89eX;4#0u%I%@2EuE# zk)FD3$*{uUzG8!hD-r}2g+&qj0S^-H6ZDzZJQ)lDt_)if?iXt6 zFEuM_&p8!G1^o>ME8G9)wDeCewRZKYTI=I)6UB-awbI=5FS>9@b*^-f=%ITTOx0Z1 zd%MBpUuhdpl+nhhC;^(7nU%^xeLdep))B=Lckf2LXzGE~{BOl|VU*ZsovllHX;Skl zi79AwI-15wIUPDH49ofkf57_Bae?r)tNT>n1oU*G>iAu~1d$HN2UX^Dc6J?HIr|Xl zw6<2=VOsWuO8qi>e5HRn+C(bewWy-eZu-!#wT~tWB&}+#9fnhFs(BloJeCx^#T^(q z2Qj>>upz1o_sjxmDB%!l4PHBTLdDASbaL(PRcUOHd8z+7aP}87-imX(q;)`846^_< z)T;6tlhw!`Dr44gD1*?nMCS7zLasOb`Dk|fS|Sy7+`~qk4Rqq=`U8)>msJE8j)6C# zw2FAy0VR^2eq4CkRBvv5&1$wnO>=x16r{i{CY=uMxEhs%-*(vmw#|Sl;CP%B^wlDC z@xMIUm+CWF{nlZ?&!oWL9=~K|RmJ4i{_+w9ax92q5Js?btV$}I;eNrJNaQbembmz^ zVj1yaS>OjV*^;z`cZ^`9rM@pcL(R;Jsg~%s@cMI2NT?8QAM_I^h3APbMNf@_B6)l&-zJR z%nK+!)iJGljc!X7P8Qd2gN%%hg-TR%pNJC#cUV=BOqA!=_@wkiL&N3nc%l(W50+PN z>TyRH#%Tu!tiG6cdlZr?n4;u301SLY6O7nT_waEdy`RFLbG`4rsPh6n4bPDtL7zam z-+Pb*Af@BmKAM1P5;6dZnVc_2ojLy|c=p;sQv<~{K^L)-B{8%3Yw>(9&3UzIb`*m? z8X{KMLW*|J;fCJg))I8m=Wb^M4h)UpR($1n-hSK{ITZI?5+*PIBX_)k)Sje0oE@v0 zyVl1iC?#Rgigx+-NFnfod02eoX^22x1)0fE5E{f}K>JPc`CCz8zU(NPDVnDolrXAH zB0Cmj7!&|s3Arxje1No&x!a-ERM1`YvxmBB94zkydd|?*2o!vN6Q|4s^6iJI3q*l{a`=Z7J zEN=Ql7IEaqTt&>kf)oY%V*yGU*%%p6pc#x)3Kgav{5K?lGKQgLWlXDd9@F-}Ihf4< zRmNF|McJ)ydx(Js;%){T!k`+%_xhiHW)=Y8p1&-|(k&qLy_uIKC(?UQY49 zPENG^JjC@#3Ce-Z!o+UiqiwV~t}{;aF};#`Btt@Gln&l8He&@oUgABrH2i0blEms& zZB}Tg6mc_{Jd5>adHqkjjXDDJw|noNX1(1tkpT9lf)XMz{_`j`0i`_CJEmorL1U;I zT!T?LJfRacah4HW`+r(M-_Cn2BV*^rr*~!h;z2QHB1=jJMo3r~xb)p-RB#~U`}*3?J7fRe$V_?GBW4tKTKwc)VeE{NKEkg zGk8#j(uJTyameE}(FyUPK-^y!&ca}KT&!By?|vYc{(5lAV{_|KI{L*=Kn1jS05QaO z=Ctdm6o_vSd1tL39}zqz^_Qj8i2%{~NE(~|#C|uSfY1m1R76Q7RI2megjT>!@W7EF zZSID!HuvLx#=vV~a1&nH7Y6OjiwkC;uk*lU+g^SneZt1;dZQ3XL7=%^uy-~fiGx6! z*LomfkN2|@%Kj4~peGwu0Eh~utPAH=MQ~7^`^+dH0rJ&2pP$2UC}KsKf;}ibg7MS3 zB#0c-dq5!RG>=bD>+c5W<^|;Ua zDE{=RV7gl>oZDiTgo7(aNR7K$5#!t0tJ^;)F=1yj+`fp@Dby?KcTFf4xyxKZxH309f<7&2V8P}~N0K_hYV>>Qa6&s=D>kQhj#|U)fCwy8>R9+5$YZW|GhXOnA zeW`W@cS;ibmF8t$Sd#;Pv!7)#b^bv2mkIF3^ZLei>++2SrTH4Zt$;DLvhbed` zq@IySqJA_oDf#DXr&MaIA3}sRBxsV7bm+sZ%|EZO>o1t}L>sY?pB}0TKpIhdiCxNu z*)(i$;qH-h3u#kTiG^{ym%d&PKtLcgy=A*SnR|$FJZ+>bGZ1%Myd?8qnmN}^JkTU= zlzD-ZJ$E0nIx%yH09XA`Y=4Fuc>6i$oW(B2KMEe?Z@d+4a46e8O_LA-Nc9iTG7W_w z!Uu`UMPiyf{yhvJEM1$8c6T*2grOetQLD`ppE=`}3R@wDm9!Nd& zTP2A~!Q?if`I!mtOLiI{W^^-o7$qRc1>b1jVKWSsh;=Geo!Z%yFdP}?+BELbRdmw` zv`cPAKS}+a_lYZ*hxSA)fw#-OwLsa)(oiwQ*xrKK2dOk2 zL%M}G-fJRq+oN*B&_R-P)*9l_x5sr-hnPWdl-wBzsqK2k|7}58l{rGK++$KE?CC62Xd!^_Nu4hwXWAXX*edd4sX7rgb*P0r0gmk$F z5c*pd>Navq7X?%1)5~x|TEXSYkI%S7>+Rtnv;!0;8R^g6`P16AmC4n2?C@5P4YK< z$UD;DHvGe|WgP4CmKZZjBJiR@mOa-Ki`>u@oHm*|&(5|FF7>+sU{I!T6mZa!LJ}el zUMhBx4HV0B#luv;#sGMxq&iY!04}3T9we~e!=aV}+2${(L#F64{P*XcznZ(CyMvW8 zibl2hoh2PmHF#!YA?RREfCA(UwBJ8M3prh3_3yFL_pz4HzHSx?d2#*VK*%Ak#Kz|e zdyyl>@^+4r-58D?l1FppA4|j@wjmKDrn~B@c!JT9oxr)x^(l*IqIqrEPcrTq6Od&U zeJwy0$YJBu@W9<@Kyk%Zr!O)556O_GPq2WyOm%aT2#E3Q zrxQT*rTq*ZiDtI~%JtfBnB*uj${F->!UA*PeC1H-_xwH}x1qGt(gO$64p z;DFPgtMHsK$JJ3;p^X=;KF=zUNE! zrk)7_$MkZ!IG=twp|S=-3aB(Z}qE+0p372I}0c|8$R8Nx>XTiAlw zZ_8Us=O{xNWw3ws!UA_W;HR2@0L1QCCc)ben=a-5cHM^kmqQCN5C!28>iol>0n#Y0 zZ28hF)+qY_E|B-xc#`GtyzI==FX9GJF?a8)Ch1Q^YcWXFoE9vJm}0Qke@(KCGB)bj zwQlaxq0J!t&sBZq9K18HS={w+Itlh%3WTTfyMy-*ec~{VFBhZnG8Zxn>@{vSx{tY@&!oVw{&Y;Z&-4N_+ zYTaug1>w40tdfo#d8CmdYkNsKQ^)U_n$T2BeizV5#K?YA1gd^C^k@3a2nr5c_NN>| z+Cnpod#c0GZFybcKC!`R<;df131$74sS0f$P+|8O=}ThL&_M?YJ45{*(H4|}tlGdX z*HY>I1`Q*8p3J&;DI`*Hvvz_fPyq#u1$34U-rZpuLPA!rTSd*2reaDc>#b!HP^<{M zS)x@8@mhp&LSt%BWI(eEImjrBOmm!DHyEl+1He$9lRgDav6~lAfiLjnA;QAo>tN{* z9B2!m0QSfC+3W(C`cf}Ro z?eY<2cjNb?Y($@^Lddkyv$9*l=^*upKD9{4S(C&@Fkuy67;((H_?+k}2E*&6BEQjs znjdZTP!0%g5{m&l0AO;o#I+AvVUDp%D%gY%X^X($B(VJhl?#EM;_D?02&vS2IB$xf zmt?YQAxFeVbwG8jL|L{^P`^o}Akgv%NhvWrc>w){N}*7wYX7Ax>Jf2#{gy)w3ii#& z#n|Pzif?m)h(J7=z7POBl?C3YdNHw9-Im)g?XrX3SAq@}m?soR_E)AdW(#|Bm*Wrt zcx0xeMPVq~gcfE7II&8FiuJ>(YwIveY`eU7bvU&UvOGdgN{lgK_#Y$i$!!Mh5FHqZ zY`dA9_k?}fO*IgkAO8CBze8go`S(`JvHyiED2j!41Pq!8$xDx=Pk?>4+fFg?iZF%) z3=os9O+w08ftv&%s_GC@qM)*2z07%2c;eu-%kjUwHvhc*zt%_NXD#1L2DeI7R`D)q zV2sBE7Y|^E%$O;k{$rrM@zP#fb%Z2^9srrQ0Bd1K4^8Fov#Cl?n!O5cSBo=;74xmc zeHqRJW09DPdl0Nz6?qBw4>O)usZ0Yz?FbX2a>9&~ROf^fGK5RwiD{R)(b0Zy+yut{ z8ZX7+={-jISgUn+g;O)k_hZSZ-6kec=W6cca%sHNGh)CDyWn}QeG9EnNB#@7pYz`7 z%x+$Z5*D#-W4gp_Y7;J|?lp%G!hENgTYDw%=l*ibJWh`vkS_&D*`i5bU1wfOeGfA(~6mj-&HyZoohJ}TPX?aJ3+qe|DsC|a+n z=I-2Zk>@+|)XlG@4I^*obo~5AZ(7)Do2{?(OaTp^%u%$W2$mfudu-lKauPQ;RS3Vcs8Q6Hzv$?WIT zU>d^)&9|poFVanEiF>@979D{82dX<~zBF@K6Ve@uz;8WHxYFe!zL&SYz^lTswV$*w zRMJy+Y3wT%Nk^_<|K$@}1zD;=+DOV(BMn;$t7B~OK*prCoV^#vez^7mf~$zcT()U0 zS*FIv&bPSiM<;2~skiM$khaD4(-e{M_m3KrPJ^80v!%G2uCBIh_FbfKJ6LtNmu;v# zoWF!88zQ(ytXGzYE-x>_(sT`HOSpXmw#vWFK4)C;+}=iG-rgHTc2*4K`b4McT&eEm zj45;sTx@$PZE$&y!7k`h=A*;RKL;PS6$VKgaO>B^LN|^(Y_mQ7(`QSs5)!t@6z!!F6Wl4y6-bP*UgiEOgcI}aPVN$ z-@|5OO<^XL1(*AJHFGvT3Uw%OF!wQelHls$LR?tLA020vJs{W=Zip>zeUIpoG!#1x=S}P`p7PMDr}n_THi|`>ODr0PmWXLO z#6pkmG1|sC4mZ1zBy@ClwNJ0&zx~q0e^G=+W%%Nay~%(q!l6)5WYy<|x*pU&1hVcb zuD*!l>O7Kw1ipZvS^~s=ks?Oek5R-LR>OBXUMLS3{aZXm)z^&+YK*Nqv< zQ)>m#GlbV>&O$eyB7ypiJI84y%F}i0VeKGL<9B6Vk{T%uLc7nDkJv2_J=n_Xs^m&P z$gWaT=nBafqo}%Q>RY`PwbIJ-YGg|t>wCmtgh}TtRYY5$8y_colkrcFEOPduCSSyQ zV`VnQ8FxM{|CJw5XeWTVT|~) zm!N%9n>L9>z99N=%WS;og{k(VW=sYBs#RKZQ3q`uyMB$!&XTX*#M)bCx=MrF(LP_~ z{Ub*0C}5cT@T&{JjnFlVvPvBq+_v&Fp9wt`&c0JVDj(HKFKhh$7{m_6zPz@M8kr41 zP3*ZbE%^8YmgJTaG-;THw?y~8T}--y{Fghu8JYY-8%C--pLjC63uKu5-tYuPsn%^Z zQF~`O5A&=&KF=?ruTDc6yp(c_p5p>$A)on6%78_5PQSg1;Y_*p%0m)o{GXIBtR6W( zm3jP3e~;ibJg?}z+|eyVPUCl4Q}_Ht9ILT|2{JDoy@oY&B?zs@%%C!_u@Em`#3tkF z&0#$^YrQ@f!MbUtIz%Jl-{kX#-^vFGVQ@o=y&0!(; zGle$9{a9ZWY1C7rba5^gK;L4GeM4P!uZOETeJ8VI^vuhI7^HVuhz){c4-{b^{nm&* zJ%jF!t$`P-d|DE9R-JOfe(mHam~}V`r=z`kc>IHovi^|Q-(ZGv%`VDR>(pa}qnl;& z`)&GM7frkCOaN|uulbQ}Z^K>ew!pUmOy)GCt#KW;b6t+iin&DNwrI`z7rA8~5v1_d!>SE^}Yw{!L58fqNQXIu9@d@SFv2 zNfbR9ziddy1r;%34@@c>-8jE&RKF;ERSb2)HWO}+$1czzzt5q2L7JAmw1k#E^m zp<*L5ZpsX9hNmH4F?zqHXu4gpxg0s|JwHX~d16LCk#xWzme=t+Z7Hx^BNX2!4GRM< zW2IO6{un(Y&ie7W6Ug6cylU`!G*GR#2@CX0igwR<<=1LbgEX?N3OGm@%1-NtMdfVxH6&ss6objj^A}00Rvo|3& z2E+{FOVp%c%)!i=5%7ps7bB^lQNnTY5wx$8=gmGi9!i&*rx7jYS;NUCW-WT0WS4tX zNwZ7hE2_#KxMR*aDbnvRfrRrdM&FGye7ov~Q&VBZ0Cm6@U%6e(rogL4q;z)}Hxvjw zjp47d{6!aKhX%0_HWI-sGZ=42+^pw7%kfcE@-2zE74td#H~Cv`lC%IZC*G5tO0w*F zY63!<>5YEXMn%brH}5UbCR7F#kqzR&T2^#~-uTG>#V?RFG>i#4V%70i_p$uynk%T@oI9bUooA_Te}75AxBA7U)TL!(EbDc1Ivv&iUAM9H zZmce?+Y{s&CM+Z$XPwjOh>a=aSCQT>H-RC2^q2uvc6bkuV~>GtmhOV{3C$XsH1YqIOJvr+$^}u^zx+6+LWyW6G-UH)loGE<2E#R7m1Dojg&M zSqt6`X$!`kK{pbIpfzwNg|)3umgcaP`HZ~dp$fe80U$8R?AFZ9Kfq@C--&wu&Do=Y z_GeT6t5kr5J=Yi9{%qR7%m1nO|9yT_^l$yY{^UPDtG`YCPgg+E|844T{Xc1i{?@-4 zRRo)3FPn~`CZM7e?~LtSpCmlO%yTymhz&X;LP5GL2gNg?tW=qeVS?HS6Cd@5m~6|q z7#J952$Qh%AkX0tWMf!~aR|p_Jt~h4o!QUN*3|~RpqSCPaHD?8s31nAJK7MY@*0Gi zE0qEbEjhySE_KZ@ixB-8rnx0r7su`O@Mo;IFV&?mS0lg?yTZ_<845ife$+CM)ve|C z8+Hy7F*v-xQ?D&MS2?LX#MBsM!V0>ZyV&w_8EKka!))Hwk`!L67s1h4AqDH=h;(8< zkZJ{YSs}0^Yd9Cbwjo5D!DoT0A!2=ett~Ael5Vt7KFOq(T>o{5k%5~N%qvIkS~NHVnbz3GP^M2Y_X&~&I3~Tl0ZnP=`i}9 zmyDL!2I(1+qCap;;|zpx03YojZd=mc)IK?ks(3J}2Hd8}BKg`%NPtLYxzN$R8)iA8 z&=|gCXhkUALEpJ#*;(Rec}LPzm7-^;BY9PtFfg@BVFmV_O_J2C8l4hl!l4P;C5$gW|Ay8_Y literal 0 HcmV?d00001 diff --git a/docs/vs-solution.png b/docs/vs-solution.png new file mode 100644 index 0000000000000000000000000000000000000000..53829938fd7225c41efabadaf39e65d17f8d0147 GIT binary patch literal 11481 zcmeHsXH*nT)MgJkh^S-`5RfDw83BPAMFa!{$r&UFNR}LDBr92R9FQcE5hO>+Ig8|+ z(~!f=^zweYzxV&1vwco?b=9p~RekH;=RSSvi`rXxG7@?c0077o6<(>|mLLE?x`}Xa z-{iymBW?>~Cj~th03fCK?}h-W>9hbqqGBT}tEOh{=<4WV?dbGKQC9Ym(Gz#pnDHsoxx!w+LToH(-pKj z0wzfmGQw7}xN5Cld=1kN*D@48V4f?e_VYg6-@o~L)3}+wi5~JACP&zY_qCaJ@YJUCW_pNSPym12|1Sef*$%84Rpf zn05G3r3iQ_UVqpobPH*Z5^Y0>lG=otN)RR^TTYuy?8`gAYs>#?8?H z)88K!5&@AAAcaf*F%GN&FxC33&WqF94rCR>SfApxmO>DEzlG#+5KZ`idz5YrIm{G5 zq`c%E03M0s0J2OJgSd~BakB)*T9yh;oPui=AV6z>!eElpm@s8+!U(;`@zK%>%SyXl z#mo0J-1j~Zud${lrLj>cY~m^p-2njltI;;BI5cK+V{Lik!=~NgY3FyW?U5s^3=3)z z+Zcg^2i$u*d^lsWzCMNP)D0%}W0;aTD=eUcIHt8g!IG>j=TTdhZm z@gWH*={~zzc9XzrxC|(~?6?%}e|{^)&UzmK&Z)4JlGdC+jPrX3_fqw!&3?R>f({p( zY}oE#I3|jFbVa1XkeBK!O7}~x8mmgTSDkw7E0*4bl?U5!Arx~WhB(fH3@8p!ZGHQj zu)5!_)&Bs36Mk2B%pHLFpYTP{lg0w~tVD#oE&w2|O(Uoo1`#G+uQ% zE`8Sh)a83sZGz%tn@bN2Kkqux5SY?tw$nC-5aYe*A)S?xXE6%6F`tJ)d%M3SWMVHBw+Z=KDk`5orj!j+yOwB=i;iwNYL${&VF!+zA{v zSlth1`mbGCE~Bm@?jyB{L{y*G>ql{5XK zOV80wZN9E6svRwA&Zzs$or<9~^Zg;wW1MgV#U7Dy(7krMfS4G0U1eR_AMAThLs^tccNlUCeR7!o-rvZp?X?!X!@N@!j5UYi~(d*kgng zNfm7sW)=8yHF9Of?%AX(bL__0_b#jvRU(QXCB@dnkjJQgyXpP2RzoS-8MdNK{Qa?d zZv2BcqL$Egyga1P%ap92?*xt(BndFj2lE!Se2)8%415Z8fkcE&1b}N{1(b>}Z&=t_u(DA1z=i`qj zrSxbD{1swIu}UUScGS{+a{k1ZDz@)If@Ferf`7v1PfaZsE$F1zPg2byE%G8`ZNL22 zTEEm*3yCMFizW*hwHgY=N{7@v)T7mg-54=l4<8kahX_q4-1Whl~y9LCD>%ilrlof9Y?A zScb@keE4wrpnMfmA4@)z_?JAJ@t$d(G28#NPrbi8bLi%j0n6acIFq!Oyl^9~7ps3) zA6!4{HoPMb#`O!d(`Ic?6 zP3`DW(DIM_{nqud9o9LciaDIM90#FRtD>GXG9QW^ADR`5ELhIjO1(F;5t_~I$n9Kf z<{pf{l9;dq?L5Ir`a!3u24pEk8z)A zILrv{q4!AkQuheJxte84#M-1WKdCs(%tcRQ!X-P~o7KZPBIf3|h3 z7_72gGEPKk@|9xxA0)Xa(P-Egd@XQ5diN)+(;w82Rga(Lq&1|?mHe}xm>-bt^eehi zd78%1YxGC@@WtuTh3<;;z-mT8T0#3s%qG9FyRn|}O%s#quXK*gDFOSQ*$#uHrhXq& zNf~L&>!EF2nJyU|q&G6OIlq~8QM#GUf670_pAemS^ZdFW6MfmT>~&0ydW&xgX@jgo znm*Hi{)KCc%Z!(Tmq#dyABo>Uz<)=E>g63`GCk%`agMDt#|&5I!gE%W~2 z4A%k!9Lh{n_oG6Fi5BnqJLX?0Zx8$?3R{x53#(y&J2hC{%P|MBVe*kH0^;TNQt_Q@1nUORGiH{;d7%8G0#)N_yZ*@tAo3%UZ{D zhxC_K7o1JIRN?fsRLw+J^V0RX?{zdA+e7`C9#VRudNID@!O0JjDxvFvk6xIWX4II=ab4f{TZxX`|fHMT34lUXXe zVCB)AD)myeyVT13KX^P1Xbn7T6`BG-;p>e?pUBQtVXxqx7=;b?wHA&Rdg7-!m`!nVE$*HGTWL3cIrE9T3w&yvn=I^2rL$zU{Qly9y9{SO6K z&E2|1qjfT~GPtBsS8D-rs|6MXnHg`P(i16jUKh&f6kDTrMom_-!>#54_^U=d``1QxuNteHu zf1es5y_ycY_84j$IOZSLve%zox}WdUc#s?@&l!0~dzPV_XD~a2*W^3!Dr|*itNqfY zX{+h*A3gHJyTqZH*m!U}ba_mEKz@?#HCtqK1~W4Q8GX~S#ZcUGL1h8OaG$?6=(X!o zw7G~`xltO{MNc1*UWE_0lzJ7Od^#*DER*bJ>t=2o@{&g7qf3Tn{SIpn>leG5m{YjW zFs$0i=(Txw!HDXG%%(3E&3_Sea>G7`6uxDfcvde|UI0Kvl<&>ExZC<8a|Lx30Pto9 z07M`FT;0-B#5Mr92>`&3DF8?&0|1TVHVP2V!p@<}&Y>>+4D)xf zsiDzSL}#{Ry3VaS(Ip;S#)2p?-ZFbEex)(E&-RU%KD(>=dkanuu9Y#cx?;Ga4P<9` zWoK6v7Ss}(KNgzjkH>PvX|5{R8RMkS69SA$p8 z;E`KWWMWEkHEO~&#%mlkQH`qR7UUX7RmaAC8XZqWp{mEnZ}lcX@CXDipNTw9aNdN87L@XAILZQ~z*GEQ1y1TnOIyy>9N^ERwSXo(b_xJx_{}&64PuHXZ0Mm}* zs~4J{GkY06t`vVWx?6HW`Fax<6BjE#*Q*PN6V$VFq=l3ZI>bAKX*9Ok&%+8m`d272 zWU1?G*;y81+ZINfe#++`-7JzS$c()ke{!%G38Rl=SMbd04^ z;(2sR?CY@}rrB@21{ayQiT*B!3dlN-gSnf@Lyxkc_IFtjK9_KF=pL%N zT4s@xO+ElpekKQfnbsA46wBx5n}(}q#>(r*wwSC!2?@ukB%aRC zjjHPXETaINaw_aO3rZ{C+l@NmMx@0GOK?=*(EzHSXiy0YKHP-Gv-j%0+*^Yoi4dLa zQD4M-JHm{p0oJU=yLze`dQIoyp?^tgE?!|ui8i!g(GYtVKcbKiQ_!1fd`Lo zR)|A>YjK{SAqd_3BdUtE!STo6ET4@n4SRWa_7E}aqP}*j&sOdaO48Yt%<{?n0m@R^ zlCb1#efRuH`>4=xI}8rs3gHw;yzO$i{HF*M_FI4eeS2}7PykWUXiMAQkS}WNX~la; zZ_eDWB;|E1?Lp@=u<_H={>ahj?QKvh?h}I zlQqe_*nnM?f$u=am}vWb_KEY3aibqj6hfQK?l{ewc(Jml9k4$&^gPe#%>Z*8w3iDP zrSi~^;^KCd8=q{SQq=GlIdS;wZ5bf!e1y@c+Ct{7B$a;h?fHUuN2#5GPK5z`s043Z zby*PiQ12Xw*;BE^$zby^|dijb^~3f-!pSYZc)cq8Uli=HL*!+)MWGS zhXt~}y&k2wMhagB&xR&B>-XZGxmH70$D6e{aV_e#XGP8&XRL$P1n=0}dmc=D6@=|H z`l!DP7T*n4eFkxMXn1Z^idxgMWJY2|AQYz&OB(=-1Z*S8F*H#zh-fc9{IG*=mxweFab~ zv^k(1R&d({bZN&EwUuRq6Zqn2b1Nc#R&pqN0uJKA+az<2BF{HEtnTC-Xnt5PoV51Y zPOVNGxFgQ;k<1(uK4%orBNL}3mB(|R8D>xXC^L2SS#sUE96!z?y37eRn~Si0qYNqU z^wgvK)!nzW3fqq_O}N0B_#&uzh*SN{CaRj8bL8%p$nsCPVle}AA2(q)iP{Rc;Ce;n zW$Wje2S+Br))alQ+C)vwP~rK%3U5Wkr*WEebMP>=JsuRKPpuO=wWUVZ0lu5e`Ubl@ z!?Y#RM-2J+ckBsA_H6EgIKVJIHeiwrrnuF`eIj)TX`zuRTv);ZlCm=eAD7^0vlC+h zBV5cyr+^H&{b49+eBBK;r{6v?-vU5pTKeCB)%AHhfj7=EY38C%f~HDS3BV-Ty{4_j zW7&?w@+b2&rjJ8xEm8B@%nPmL2!9Kw1;$euPv`m>eu(6!UC-Cir?YG>oVHLLdwebLTClW>0Rz;$Osi1OFsO~SRkt~?x0Hme-}4JL(~6U3ZN zJ7fmjW?5B2@dS9E7lD)AvOoFZp2YCL&+~oFfnhVQ?nRQ1pk7)8=K|7Mo#s)JU%$g? zvrA9`Y~c$9KPl6){aj%2*F6f(g|gKaPrthCubtV~m ztTPfwoi9_ky0>a#Gd}-yJcFu{qgy@CcN`o_+jYjbqh*(i{vnha|6(k+D5#rLS0Y_q z#EK{%{)v6ZCFe;C9Bq)ck~+Q}2bC83>cMS__`>u;7U1bx@3{po;>k^Z2{!C`QZYwE zO;hV3#boh3KF9Ops7DOUnpPIG*=BuCp9;65N!JDr%=E)GqKlJ!Oa=)=wcxL~u;%WX zxG-Fo!5=g*6%f6PapE!C-OkUSp+7nYopuc!ox1&+6Q{io&+RHFuNH2aIJeHpfu|_r z?0v9e5`kUJAS0hmLB5BnEP{t2t1@#{bjkic->zf`77X2bW^{280k|Lg#zV_*@$?Ta z8Kwq8;#En1X!xwIpdf#T#b(!!zQ?)#1V?e|T8v?Jm)dO|o9h@RKTURPV}85D1*q0{M) z{bqmBCkLNn*|V(c+LuFhL!WH%#tRNxd5v`@8@ z;xJ#VayGuS?t5a{t$lXC4Q+fmXSJlmCW_6zsCu8e>j5V_J_$be=PVu2yonXZ9!fmC zOgfk#gus0W*b92?LXmZ)88YSfK zsZ<=Qc*BWC{4OaOY`T0Q=rLFoAS%W4gG%*!?f4B=?yF6Or6pmOJp*o)y}i-P{;|Rz zJi7af{qkWf!SvFrO=ZwE-mF=PLh1ESPz2tS|Sgf@Btz;mh zIrIygP|9xz8Dg#&E9n>L^HEfrkmv-kG?!7stjH7vG<<>~lwth8MI(LQTHP4a0OA=ZMd0JVOsxqbuiih1Z$Wz0nDN?? zb8L$Hk&pC(oGm`9jSU@~^y8#n2E%qc(6;q}{sbV5>rMRH$t45i)$u{$ELT`B0bs>k z)je<0YQ9N%HDIf|Ks;62w~yQ}$YsK@#Zze5}To3&R5 z!hn-_#3~q&{h1aV4JcdxBfijH6a)o@X)#DRdi5of@=!4CwMSWc-QDVz=1@5LqOZ8< zR+99yol=we3zya3<2poqEtFc^)%EA#|8e8c`8^F_aKtxOGm zc{Aix0x0IbmTHC7`C0m9Ir1dEwzw%c#Ys#Y9Q-jlR_5>9pz-|D?KvV~kk%f@bfL0Q z@twZKPbDj!JsnW3=66+(W`R~bwL}@@>lo|!s3VoxjVh1JU9Dy;Z%A79(g)KJpESXl zjzo9kejAJaeSxj1KVuKRVUgbYBYpL(?)nGl+`DzCYM?%n(7Om2&AC$9AWWizdJ_Xv z90#433vt`zPlf$Yzg-p8b$L(tU5jXBI+ThhKq?{jo6&bMowCha_rK+eDe2j7FgmBVNCki&B6o#;;k1av%P3 z_&J;V35f7Z7)z@qQ?tX5`LyozjR2M!&=-lnw&(+YuKb7J3mGVu9D=2?rNYYl36Fe{ zV1d^$IBKx`0H=|tHJ=lMgQ!&fq2~I3x<9*N_VkD-ez1WWCa?WxdXg8|r=zNdKiC!2 zA5PH*6^pyA$2U8@H>=k)7jv5FtmAHi6xI`M>kVsHJuC(G$A{t;OF-8lk#m&c4)ONO zNRZM5{30f9HTJd$JjykZZ@1RUBRZdn3FpMxs?{*%Z&ttSM}9hZc+bvP2;@WuA-ABy zkqxM}|qF;Pmgg*jvm1E}C*BK_q z$#DJa;adhEDwSInjoN~y*C8*^sugeWOmD+%GB29vG)INLOUXLOInT;3q>IWMkXBx( zb~yP(t@8MobOKKBw?NoKrs_l8S#_DmM>>8M_El)x^_||5NJO1b3M$$;4Ov+FF{1JO zums%u_h$!6=y#0}?Qcx2Ia^I~Dr&h4D-CG2IDOu-Cdk}qeGnn#&yVR2hFwQ{SvGp? z+8>s^*=9Tlx+>y!E$czi9jix3ZKUlFd&n8y(EI?4d4iZ|OA0KRfI=jPJ47BrI$P-C z8zRNn<|N$h8z`y9@LLA41zjG458PT@BSWmRRQB=|IWb;vABT`y2xPmBhTGi{I0YYL z!!seIUUTv>al?j}!5whE6@WUkqA>L2S| zG1=03$u9|n6OVlTo6g_YcE$RK#jCN)o14sZr77S+K9#nAHp*mbadVds+@{IGH+7N* z#58$m-|H_w1c{`g~jiN7xtUZ6nrAIu23`vad`EG z8HT9h1xjM_mb&45QJ;rbB0$y?WN-#Zq*z#zizS9GbE1_BKsGECXSKys8%J_YnCS^o zf4xkbJs7asvc{(c42)M3h_Ma`n@(uSp)vf^V5lB(Pkl!hG(6E?UTepQNx2Y3;S>hI=N&D|N65?8Ov&bn;SE`$|#@lB2b8aDe2H z-7K!sy2(Smdd}Nkw_o<3vOdv{;?#NfzeqD$YF(*mL7EU5ru!d z>g5-#>?t#Fo1Ss2=_&2b4Nwvw%F9M3u>K4XxaZn^lvM_rmxb1~4(lJ)c6yrTv!739 zo~ffhP;EvynonQWzt%ku9*5I^dZ-(>!?2L2D>xG{Q4=^z^QHYSHh35MoS&DFoiU#s z`FSR^{}jqa{{yqjN@`?j+3NJ`dlgZB$RTx`=5697x}9Fm=6;B(cl!X$U9EcTC!lv!?a(aE*rMu zo*%%!LxB0IGTb-s*mtZUPr{>eeX|s~Yjn}-wl5iIK9BosWpxJfqyKgE#e5M)WcaBU zAKUFC(%bVNDx7=WC~q6uI3+kyRw2UqvBk`(_@@728<&FTaraP}A&Z1Nlfu|r*JO!} zZ2!o-6bZ4L_}7Z*@fOJ>_1M*c=bya3mOKH31~ZhddMZZv+(|s1uH%k1(ZlL>e&y48 zn~F49PrI1&hya}roz_i#e(?|%zv&PiXN=p_4cdZkCc(?=HtuJco*WelUC2S;pyKFo zkZ#GRwtL6$`!_z6G^plI%yRA7oD`SF5}-~8x!ru zRlnhiunG8tW$dvm%_uw!OQGriVmn{3{i+vz3nKpC!P`ZOUq?rS0m!@ zf-++`XCZ9Bl!a&|-F8MBf!pqTi3p6HwO;KFJbM>Gc{)1Ul zqB+Jpy7ih>V05!Kq-RxhO36J++EG25pW+2mnFL%b;6zN;_$66!(q4>`lYiE@;W1lJ zN!KD)gDUM^(5=j&O9l}o^ox!<3p3mehTbJG-*3F}ubAoWOv1+G3gXtfJ+rTmcwYue z+<>)#ZMx5gOHJL>D98Ib=IMqCcpg7f*-41qm8`mlSxo;QLljFVIO>#VT9nWM#rtXW zpPhr7IqODw%=>&kNxxj?rZ&S^{boxHApAqgO_y>V{e4aTy%C15@)Y^P$l+5mO11`n|7F{^FV?NaamLD-d;ru!a7XG@J z@NhcvOg?niZd`CQ?H>C-=%I>4LRY-csi$KfckDqX5_vP-sOL9ueZfe}|JJ&|Bzn;U zJtxn+-UlzNJG`um-fU+S0-2GQ{2v4#c~f}eH+eLqW7RmG-1$W8ah3Ht|6vo0Zt9|7(7{J3130d*V%l|Y{A+5WC2&k?wTB3#aw#y$0m91EXev_sKEfg zgIy-XVtE|?zXjYwMyg?ne)?N{ixLG+-I0Sn&e=rc@b<(oP<3q^8XgY4ye3ZpAN+4{ z{WWpQ{G;FZx4#p4L^93S(8{}78;$2`+U8;SNC_0_JV<(+`15sZ<0xD`N9&=dd1}Cv zM+zv-*5!~hyd_8(*n6}EUGQn+Z$}MFogq*9cT<1@kpTXxyalO`;@=rQ5wk-$vm)od)sb7Q3d-#VkOKE|^ zwzZdI%-=+^@B31}0_)010#n}V4?2@d`I6j^oAZaY(frq*eGyPFna)<+`lsL?q%eup zLuAw+#&W9?ilz7j&FYzgw{H?sez{f^#~@hWfy4#O_fo+B^3f@BOU)+;MXKckP}+So zBa}$bzDv<&Svp`7R424KA|@-DVX5T2R*ZgFiy<}Hm#;V%HfBzT-8cMHgo0f}g|&eA zMbg{E&mKbFKI97EIaBp`&X!@a>e2*_3cEYzM^uDP4#1adeiI znEMj7q+g6dnpSg4D+U&QT)1g@hf>uGy;S1Piyh)u+ZS|H9>L8Ya$We~|17c+w>9(q3+iOT=f&%j4-{x(J+12ORH_gD6UXi8I09-$+^^Xj1|*IRSSn>COnL+ zX{7uH$DsvQ{_=mg$t#ffJ2uCYYviFvy=?3-|HN$Q`9BmhuGWSKfq0cd7gW8Obm>9~ z{ALfnZ1PJU+#k4g5%FdH-#o$7?@Ut0dqO-$L#Zb(b+#N#4n&$~e;EHQ9GhSLHyr-& zboZVgmwIRidKlVPgemW}O<s1Y0NbHh7`igt2j{x}jPdTnB+X&C`)N7q8n$c{o4An!CfCqag;&oL%7S_ns(jZs@Pk`` z65%$c5JgMnEH1`f@!t}zAb8;!{Ptg94_r<7(2}oe@f)6vh02f~4ZK%DDlme>DqyS< zjmZr%6pZ##$~Ftt)4~5$+E?6=Q^u!hXTrJ)eQ~Y2V~`#>+kHWQ8xw;{K_*UBNd@~# zVUYowG8WskncpPG)^Gv0Vo1tQ+VFwk`>+T2 zxRBc;+JH*-w|7so^>MPue=ffYm!>M0f3d2g@cD(!3c@#w9Dz4UwuLLo&3w=s90E`l|3~nnhLYPJ0z(peenP4R8k-j=Jpo!aX1mXf$5e0w7 UvTf!6I}}%xd;6;NrD?$b0w)TNtN;K2 literal 0 HcmV?d00001 diff --git a/junction/Averager.cpp b/junction/Averager.cpp new file mode 100644 index 0000000..8ee0d00 --- /dev/null +++ b/junction/Averager.cpp @@ -0,0 +1,30 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include + +namespace junction { + +double Averager::getStdDev() { + finalize(); + double avg = getAverage(); + double dev = 0; + for (ureg i = 0; i < m_values.size(); i++) { + double diff = m_values[i] - avg; + dev += diff * diff; + } + return sqrt(dev / m_values.size()); +} + +} // namespace junction diff --git a/junction/Averager.h b/junction/Averager.h new file mode 100644 index 0000000..0c30fb2 --- /dev/null +++ b/junction/Averager.h @@ -0,0 +1,63 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_AVERAGER_H +#define JUNCTION_AVERAGER_H + +#include +#include +#include + +namespace junction { + +class Averager { +private: + std::vector m_values; + bool m_finalized; + +public: + Averager() : m_finalized(false) { + } + + void add(double value) { + TURF_ASSERT(!m_finalized); + m_values.push_back(value); + } + + ureg getNumValues() const { + return m_values.size(); + } + + void finalize(ureg bestValueCount = 0) { + if (!m_finalized) { + std::sort(m_values.begin(), m_values.end()); + if (bestValueCount) + m_values.resize(bestValueCount); + m_finalized = true; + } + } + + double getAverage() { + finalize(); + double sum = 0; + for (ureg i = 0; i < m_values.size(); i++) { + sum += m_values[i]; + } + return sum / m_values.size(); + } + + double getStdDev(); +}; + +} // namespace junction + +#endif // JUNCTION_AVERAGER_H diff --git a/junction/ConcurrentMap_Grampa.cpp b/junction/ConcurrentMap_Grampa.cpp new file mode 100644 index 0000000..2acb10f --- /dev/null +++ b/junction/ConcurrentMap_Grampa.cpp @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include + +namespace junction { + +TURF_TRACE_DEFINE_BEGIN(ConcurrentMap_Grampa, 27) // autogenerated by TidySource.py +TURF_TRACE_DEFINE("[locateTable] flattree lookup redirected") +TURF_TRACE_DEFINE("[createInitialTable] race to create initial table") +TURF_TRACE_DEFINE("[publishTableMigration] called") +TURF_TRACE_DEFINE("[publishTableMigration] replacing single root with single root") +TURF_TRACE_DEFINE("[publishTableMigration] replacing flattree with single root") +TURF_TRACE_DEFINE("[publishTableMigration] replacing single root with flattree") +TURF_TRACE_DEFINE("[publishTableMigration] publishing subtree to existing flattree") +TURF_TRACE_DEFINE("[publishTableMigration] existing flattree too small") +TURF_TRACE_DEFINE("[publishTableMigration] redirected") +TURF_TRACE_DEFINE("[publishTableMigration] recovering from partial publish") +TURF_TRACE_DEFINE("[Mutator] find constructor called") +TURF_TRACE_DEFINE("[Mutator] find was redirected") +TURF_TRACE_DEFINE("[Mutator] insert constructor called") +TURF_TRACE_DEFINE("[Mutator] insert was redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] called") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] exchanged Value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] detected race to write value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] racing write inserted new value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] was redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] was re-redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] overflow after redirect") +TURF_TRACE_DEFINE("[Mutator::eraseValue] called") +TURF_TRACE_DEFINE("[Mutator::eraseValue] detected race to write value") +TURF_TRACE_DEFINE("[Mutator::eraseValue] was redirected") +TURF_TRACE_DEFINE("[Mutator::eraseValue] was re-redirected") +TURF_TRACE_DEFINE("[get] called") +TURF_TRACE_DEFINE("[get] was redirected") +TURF_TRACE_DEFINE_END(ConcurrentMap_Grampa, 27) + +} // namespace junction diff --git a/junction/ConcurrentMap_Grampa.h b/junction/ConcurrentMap_Grampa.h new file mode 100644 index 0000000..d7fa979 --- /dev/null +++ b/junction/ConcurrentMap_Grampa.h @@ -0,0 +1,571 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_CONCURRENTMAP_GRAMPA_H +#define JUNCTION_CONCURRENTMAP_GRAMPA_H + +#include +#include +#include +#include +#include + +namespace junction { + +TURF_TRACE_DECLARE(ConcurrentMap_Grampa, 27) + +template , class VT = DefaultValueTraits> +class ConcurrentMap_Grampa { +public: + typedef K Key; + typedef V Value; + typedef KT KeyTraits; + typedef VT ValueTraits; + typedef typename turf::util::BestFit::Unsigned Hash; + typedef details::Grampa Details; + +private: + turf::Atomic m_root; + + bool locateTable(typename Details::Table*& table, ureg& sizeMask, Hash hash) { + ureg root = m_root.load(turf::Consume); + if (root & 1) { + typename Details::FlatTree* flatTree = (typename Details::FlatTree*) (root & ~ureg(1)); + for (;;) { + ureg leafIdx = (hash >> flatTree->safeShift); + table = flatTree->getTables()[leafIdx].load(turf::Relaxed); + if (ureg(table) != Details::RedirectFlatTree) { + sizeMask = (Details::LeafSize - 1); + return true; + } + TURF_TRACE(ConcurrentMap_Grampa, 0, "[locateTable] flattree lookup redirected", uptr(flatTree), uptr(leafIdx)); + typename Details::FlatTreeMigration* migration = Details::getExistingFlatTreeMigration(flatTree); + migration->run(); + migration->m_completed.wait(); + flatTree = migration->m_destination; + } + } else { + if (!root) + return false; + table = (typename Details::Table*) root; + sizeMask = table->sizeMask; + return true; + } + } + + void createInitialTable(ureg initialSize) { + if (!m_root.load(turf::Relaxed)) { + // This could perform DCLI, but let's avoid needing a mutex instead. + typename Details::Table* table = Details::Table::create(initialSize, 0, sizeof(Hash) * 8); + if (m_root.compareExchange(uptr(NULL), uptr(table), turf::Release)) { + TURF_TRACE(ConcurrentMap_Grampa, 1, "[createInitialTable] race to create initial table", uptr(this), 0); + table->destroy(); + } + } + } + +public: + ConcurrentMap_Grampa(ureg initialSize = 0) : m_root(uptr(NULL)) { + // FIXME: Support initialSize argument + TURF_UNUSED(initialSize); + } + + ~ConcurrentMap_Grampa() { + ureg root = m_root.loadNonatomic(); + if (root & 1) { + typename Details::FlatTree* flatTree = (typename Details::FlatTree*) (root & ~ureg(1)); + ureg size = (Hash(-1) >> flatTree->safeShift) + 1; + typename Details::Table* lastTableGCed = NULL; + for (ureg i = 0; i < size; i++) { + typename Details::Table* t = flatTree->getTables()[i].loadNonatomic(); + TURF_ASSERT(ureg(t) != Details::RedirectFlatTree); + if (t != lastTableGCed) { + t->destroy(); + lastTableGCed = t; + } + } + flatTree->destroy(); + } else if (root) { + typename Details::Table* t = (typename Details::Table*) root; + t->destroy(); + } + } + + // publishTableMigration() is called by exactly one thread from Details::TableMigration::run() + // after all the threads participating in the migration have completed their work. + // There are no racing writes to the same range of hashes. + void publishTableMigration(typename Details::TableMigration* migration) { + TURF_TRACE(ConcurrentMap_Grampa, 2, "[publishTableMigration] called", uptr(migration), 0); + if (migration->m_safeShift == 0) { + // This TableMigration replaces the entire map with a single table. + TURF_ASSERT(migration->m_baseHash == 0); + TURF_ASSERT(migration->m_numDestinations == 1); + ureg oldRoot = m_root.loadNonatomic(); // There are no racing writes to m_root. + // Store the single table in m_root directly. + typename Details::Table* newTable = migration->getDestinations()[0]; + m_root.store(uptr(newTable), turf::Release); // Make table contents visible + newTable->isPublished.signal(); + if ((oldRoot & 1) == 0) { + TURF_TRACE(ConcurrentMap_Grampa, 3, "[publishTableMigration] replacing single root with single root", uptr(migration), 0); + // If oldRoot is a table, it must be the original source of the migration. + TURF_ASSERT((typename Details::Table*) oldRoot == migration->getSources()[0].table); + // Don't GC it here. The caller will GC it since it's a source of the TableMigration. + } else { + TURF_TRACE(ConcurrentMap_Grampa, 4, "[publishTableMigration] replacing flattree with single root", uptr(migration), 0); + // The entire previous flattree is being replaced. + Details::garbageCollectFlatTree((typename Details::FlatTree*) (oldRoot & ~ureg(1))); + } + // Caller will GC the TableMigration. + } else { + // We are either publishing a subtree of one or more tables, or replacing the entire map with multiple tables. + // In either case, there will be a flattree after this function returns. + TURF_ASSERT(migration->m_safeShift < sizeof(Hash) * 8); // If m_numDestinations > 1, some index bits must remain after shifting + ureg oldRoot = m_root.load(turf::Consume); + if ((oldRoot & 1) == 0) { + // There's no flattree yet. This means the TableMigration is publishing the full range of hashes. + TURF_ASSERT(migration->m_baseHash == 0); + TURF_ASSERT((Hash(-1) >> migration->m_safeShift) == (migration->m_numDestinations - 1)); + // The oldRoot should be the original source of the migration. + TURF_ASSERT((typename Details::Table*) oldRoot == migration->getSources()[0].table); + // Furthermore, it is guaranteed that there are no racing writes to m_root. + // Create a new flattree and store it to m_root. + TURF_TRACE(ConcurrentMap_Grampa, 5, "[publishTableMigration] replacing single root with flattree", uptr(migration), 0); + typename Details::FlatTree* flatTree = Details::FlatTree::create(migration->m_safeShift); + typename Details::Table* prevTable = NULL; + for (ureg i = 0; i < migration->m_numDestinations; i++) { + typename Details::Table* newTable = migration->getDestinations()[i]; + flatTree->getTables()[i].storeNonatomic(newTable); + if (newTable != prevTable) { + newTable->isPublished.signal(); + prevTable = newTable; + } + } + m_root.store(uptr(flatTree) | 1, turf::Release); // Ensure visibility of flatTree->tables + // Caller will GC the TableMigration. + // Caller will also GC the old oldRoot since it's a source of the TableMigration. + } else { + // There is an existing flattree, and we are publishing one or more tables to it. + // Attempt to publish the subtree in a loop. + // The loop is necessary because we might get redirected in the middle of publishing. + TURF_TRACE(ConcurrentMap_Grampa, 6, "[publishTableMigration] publishing subtree to existing flattree", uptr(migration), 0); + typename Details::FlatTree* flatTree = (typename Details::FlatTree*) (oldRoot & ~ureg(1)); + ureg subTreeEntriesPublished = 0; + typename Details::Table* tableToReplace = migration->getSources()[0].table; + // Wait here so that we only replace tables that are fully published. + // Otherwise, there will be a race between a subtree and its own children. + // (If all ManualResetEvent objects supported isPublished(), we could add a TURF_TRACE counter for this. + // In previous tests, such a counter does in fact get hit.) + tableToReplace->isPublished.wait(); + typename Details::Table* prevTable = NULL; + for (;;) { + publishLoop: + if (migration->m_safeShift < flatTree->safeShift) { + // We'll need to migrate to larger flattree before publishing our new subtree. + // First, try to create a FlatTreeMigration with the necessary properties. + // This will fail if an existing FlatTreeMigration has already been created using the same source. + // In that case, we'll help complete the existing FlatTreeMigration, then we'll retry the loop. + TURF_TRACE(ConcurrentMap_Grampa, 7, "[publishTableMigration] existing flattree too small", uptr(migration), 0); + typename Details::FlatTreeMigration* flatTreeMigration = Details::createFlatTreeMigration(*this, flatTree, migration->m_safeShift); + tableToReplace->jobCoordinator.runOne(flatTreeMigration); + flatTreeMigration->m_completed.wait(); // flatTreeMigration->m_destination becomes entirely visible + flatTree = flatTreeMigration->m_destination; + // The FlatTreeMigration has already been GC'ed by the last worker. + // Retry the loop. + } else { + ureg repeat = ureg(1) << (migration->m_safeShift - flatTree->safeShift); + ureg dstStartIndex = migration->m_baseHash >> flatTree->safeShift; + // The subtree we're about to publish fits inside the flattree. + TURF_ASSERT(dstStartIndex + migration->m_numDestinations * repeat - 1 <= Hash(-1) >> flatTree->safeShift); + // If a previous attempt to publish got redirected, resume publishing into the new flattree, + // starting with the first subtree entry that has not yet been fully published, as given by subTreeEntriesPublished. + // (Note: We could, in fact, restart the publish operation starting at entry 0. That would be valid too. + // We are the only thread that can modify this particular range of the flattree at this time.) + turf::Atomic* dstLeaf = flatTree->getTables() + dstStartIndex + (subTreeEntriesPublished * repeat); + typename Details::Table** subFlatTree = migration->getDestinations(); + while (subTreeEntriesPublished < migration->m_numDestinations) { + typename Details::Table* srcTable = subFlatTree[subTreeEntriesPublished]; + for (ureg r = repeat; r > 0; r--) { + typename Details::Table* probeTable = tableToReplace; + while (!dstLeaf->compareExchangeStrong(probeTable, srcTable, turf::Relaxed)) { + if (ureg(probeTable) == Details::RedirectFlatTree) { + // We've been redirected. + // Help with the FlatTreeMigration, then try again. + TURF_TRACE(ConcurrentMap_Grampa, 8, "[publishTableMigration] redirected", uptr(migration), uptr(dstLeaf)); + typename Details::FlatTreeMigration* flatTreeMigration = Details::getExistingFlatTreeMigration(flatTree); + tableToReplace->jobCoordinator.runOne(flatTreeMigration); + flatTreeMigration->m_completed.wait(); // flatTreeMigration->m_destination becomes entirely visible + flatTree = flatTreeMigration->m_destination; + goto publishLoop; + } + // The only other possibility is that we were previously redirected, and the subtree entry got partially published. + TURF_TRACE(ConcurrentMap_Grampa, 9, "[publishTableMigration] recovering from partial publish", uptr(migration), 0); + TURF_ASSERT(probeTable == srcTable); + } + // The caller will GC the table) being replaced them since it's a source of the TableMigration. + dstLeaf++; + } + if (prevTable != srcTable) { + srcTable->isPublished.signal(); + prevTable = srcTable; + } + subTreeEntriesPublished++; + } + // We've successfully published the migrated sub-flattree. + // Caller will GC the TableMigration. + break; + } + } + } + } + } + + void publishFlatTreeMigration(typename Details::FlatTreeMigration* migration) { + // There are no racing writes. + // Old root must be the migration source (a flattree). + TURF_ASSERT(m_root.loadNonatomic() == (ureg(migration->m_source) | 1)); + // Publish the new flattree, making entire table contents visible. + m_root.store(uptr(migration->m_destination) | 1, turf::Release); + // Don't GC the old flattree. The FlatTreeMigration will do that, since it's a source. + } + + // A Mutator represents a known cell in the hash table. + // It's meant for manipulations within a temporary function scope. + // Obviously you must not call QSBR::Update while holding a Mutator. + // Any operation that modifies the table (exchangeValue, eraseValue) + // may be forced to follow a redirected cell, which changes the Mutator itself. + // Note that even if the Mutator was constructed from an existing cell, + // exchangeValue() can still trigger a resize if the existing cell was previously marked deleted, + // or if another thread deletes the key between the two steps. + class Mutator { + private: + friend class ConcurrentMap_Grampa; + + ConcurrentMap_Grampa& m_map; + typename Details::Table* m_table; + ureg m_sizeMask; + typename Details::Cell* m_cell; + Value m_value; + + // Constructor: Find existing cell + Mutator(ConcurrentMap_Grampa& map, Key key, bool) : m_map(map), m_value(Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_Grampa, 10, "[Mutator] find constructor called", uptr(map.m_root.load(turf::Relaxed)), uptr(key)); + Hash hash = KeyTraits::hash(key); + for (;;) { + if (!m_map.locateTable(m_table, m_sizeMask, hash)) + return; + m_cell = Details::find(hash, m_table, m_sizeMask); + if (!m_cell) + return; + m_value = m_cell->value.load(turf::Consume); + if (m_value != Value(ValueTraits::Redirect)) + return; // Found an existing value + // We've encountered a Redirect value. Help finish the migration. + TURF_TRACE(ConcurrentMap_Grampa, 11, "[Mutator] find was redirected", uptr(m_table), 0); + m_table->jobCoordinator.participate(); + // Try again using the latest root. + } + } + + // Constructor: Insert cell + Mutator(ConcurrentMap_Grampa& map, Key key) : m_map(map), m_value(Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_Grampa, 12, "[Mutator] insert constructor called", uptr(map.m_root.load(turf::Relaxed)), uptr(key)); + Hash hash = KeyTraits::hash(key); + for (;;) { + if (!m_map.locateTable(m_table, m_sizeMask, hash)) { + m_map.createInitialTable(Details::MinTableSize); + } else { + ureg overflowIdx; + switch (Details::insert(hash, m_table, m_sizeMask, m_cell, overflowIdx)) { // Modifies m_cell + case Details::InsertResult_InsertedNew: { + // We've inserted a new cell. Don't load m_cell->value. + return; + } + case Details::InsertResult_AlreadyFound: { + // The hash was already found in the table. + m_value = m_cell->value.load(turf::Consume); + if (m_value == Value(ValueTraits::Redirect)) { + // We've encountered a Redirect value. + TURF_TRACE(ConcurrentMap_Grampa, 13, "[Mutator] insert was redirected", uptr(m_table), uptr(m_value)); + break; // Help finish the migration. + } + return; // Found an existing value + } + case Details::InsertResult_Overflow: { + Details::beginTableMigration(m_map, m_table, overflowIdx); + break; + } + } + // A migration has been started (either by us, or another thread). Participate until it's complete. + m_table->jobCoordinator.participate(); + } + // Try again using the latest root. + } + } + + public: + Value getValue() const { + // Return previously loaded value. Don't load it again. + return m_value; + } + + Value exchangeValue(Value desired) { + TURF_ASSERT(desired != Value(ValueTraits::NullValue)); + TURF_ASSERT(m_cell); // Cell must have been found or inserted + TURF_TRACE(ConcurrentMap_Grampa, 14, "[Mutator::exchangeValue] called", uptr(m_table), uptr(m_value)); + for (;;) { + Value oldValue = m_value; + if (m_cell->value.compareExchangeStrong(m_value, desired, turf::ConsumeRelease)) { + // Exchange was successful. Return previous value. + TURF_TRACE(ConcurrentMap_Grampa, 15, "[Mutator::exchangeValue] exchanged Value", uptr(m_value), uptr(desired)); + Value result = m_value; + m_value = desired; // Leave the mutator in a valid state + return result; + } + // The CAS failed and m_value has been updated with the latest value. + if (m_value != Value(ValueTraits::Redirect)) { + TURF_TRACE(ConcurrentMap_Grampa, 16, "[Mutator::exchangeValue] detected race to write value", uptr(m_table), uptr(m_value)); + if (oldValue == Value(ValueTraits::NullValue) && m_value != Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_Grampa, 17, "[Mutator::exchangeValue] racing write inserted new value", uptr(m_table), uptr(m_value)); + } + // There was a racing write (or erase) to this cell. + // Pretend we exchanged with ourselves, and just let the racing write win. + return desired; + } + // We've encountered a Redirect value. Help finish the migration. + TURF_TRACE(ConcurrentMap_Grampa, 18, "[Mutator::exchangeValue] was redirected", uptr(m_table), uptr(m_value)); + Hash hash = m_cell->hash.load(turf::Relaxed); + for (;;) { + // Help complete the migration. + m_table->jobCoordinator.participate(); + // Try again in the latest table. + // FIXME: locateTable() could return false if the map is concurrently cleared (m_root set to 0). + // This is not concern yet since clear() is not implemented. + bool exists = m_map.locateTable(m_table, m_sizeMask, hash); + TURF_ASSERT(exists); + TURF_UNUSED(exists); + m_value = Value(ValueTraits::NullValue); + ureg overflowIdx; + switch (Details::insert(hash, m_table, m_sizeMask, m_cell, overflowIdx)) { // Modifies m_cell + case Details::InsertResult_AlreadyFound: + m_value = m_cell->value.load(turf::Consume); + if (m_value == Value(ValueTraits::Redirect)) { + TURF_TRACE(ConcurrentMap_Grampa, 19, "[Mutator::exchangeValue] was re-redirected", uptr(m_table), uptr(m_value)); + break; + } + goto breakOuter; + case Details::InsertResult_InsertedNew: + goto breakOuter; + case Details::InsertResult_Overflow: + TURF_TRACE(ConcurrentMap_Grampa, 20, "[Mutator::exchangeValue] overflow after redirect", uptr(m_table), overflowIdx); + Details::beginTableMigration(m_map, m_table, overflowIdx); + break; + } + // We were redirected... again + } + breakOuter: + ; + // Try again in the new table. + } + } + + void setValue(Value desired) { + exchangeValue(desired); + } + + Value eraseValue() { + TURF_ASSERT(m_cell); // Cell must have been found or inserted + TURF_TRACE(ConcurrentMap_Grampa, 21, "[Mutator::eraseValue] called", uptr(m_table), uptr(m_value)); + for (;;) { + if (m_value == Value(ValueTraits::NullValue)) + return m_value; + TURF_ASSERT(m_cell); // m_value is non-NullValue, therefore cell must have been found or inserted. + if (m_cell->value.compareExchangeStrong(m_value, Value(ValueTraits::NullValue), turf::Consume)) { + // Exchange was successful and a non-NullValue value was erased and returned by reference in m_value. + TURF_ASSERT(m_value != Value(ValueTraits::NullValue)); // Implied by the test at the start of the loop. + Value result = m_value; + m_value = Value(ValueTraits::NullValue); // Leave the mutator in a valid state + return result; + } + // The CAS failed and m_value has been updated with the latest value. + TURF_TRACE(ConcurrentMap_Grampa, 22, "[Mutator::eraseValue] detected race to write value", uptr(m_table), uptr(m_value)); + if (m_value != Value(ValueTraits::Redirect)) { + // There was a racing write (or erase) to this cell. + // Pretend we erased nothing, and just let the racing write win. + return Value(ValueTraits::NullValue); + } + // We've been redirected to a new table. + TURF_TRACE(ConcurrentMap_Grampa, 23, "[Mutator::eraseValue] was redirected", uptr(m_table), uptr(m_cell)); + Hash hash = m_cell->hash.load(turf::Relaxed); // Re-fetch hash + for (;;) { + // Help complete the migration. + m_table->jobCoordinator.participate(); + // Try again in the latest table. + if (!m_map.locateTable(m_table, m_sizeMask, hash)) + m_cell = NULL; + else + m_cell = Details::find(hash, m_table, m_sizeMask); + if (!m_cell) { + m_value = Value(ValueTraits::NullValue); + return m_value; + } + m_value = m_cell->value.load(turf::Relaxed); + if (m_value != Value(ValueTraits::Redirect)) + break; + TURF_TRACE(ConcurrentMap_Grampa, 24, "[Mutator::eraseValue] was re-redirected", uptr(m_table), uptr(m_cell)); + } + } + } + }; + + Mutator insert(Key key) { + return Mutator(*this, key); + } + + Mutator find(Key key) { + return Mutator(*this, key, false); + } + + // Lookup without creating a temporary Mutator. + Value get(Key key) { + Hash hash = KeyTraits::hash(key); + TURF_TRACE(ConcurrentMap_Grampa, 25, "[get] called", uptr(this), uptr(hash)); + for (;;) { + typename Details::Table* table; + ureg sizeMask; + if (!locateTable(table, sizeMask, hash)) + return Value(ValueTraits::NullValue); + typename Details::Cell* cell = Details::find(hash, table, sizeMask); + if (!cell) + return Value(ValueTraits::NullValue); + Value value = cell->value.load(turf::Consume); + if (value != Value(ValueTraits::Redirect)) + return value; // Found an existing value + // We've been redirected to a new table. Help with the migration. + TURF_TRACE(ConcurrentMap_Grampa, 26, "[get] was redirected", uptr(table), 0); + table->jobCoordinator.participate(); + // Try again in the new table. + } + } + + Value insert(Key key, Value desired) { + Mutator iter(*this, key); + return iter.exchangeValue(desired); + } + + Value exchange(Key key, Value desired) { + Mutator iter(*this, key); + return iter.exchangeValue(desired); + } + + Value erase(Key key) { + Mutator iter(*this, key, false); + return iter.eraseValue(); + } + + // The easiest way to implement an Iterator is to prevent all Redirects. + // The currrent Iterator does that by forbidding concurrent inserts. + // To make it work with concurrent inserts, we'd need a way to block TableMigrations as the Iterator visits each table. + // FlatTreeMigrations, too. + class Iterator { + private: + typename Details::FlatTree* m_flatTree; + ureg m_flatTreeIdx; + typename Details::Table* m_table; + ureg m_idx; + Key m_hash; + Value m_value; + + public: + Iterator(ConcurrentMap_Grampa& map) { + // Since we've forbidden concurrent inserts (for now), nonatomic would suffice here, but let's plan ahead: + ureg root = map.m_root.load(turf::Consume); + if (root & 1) { + m_flatTree = (typename Details::FlatTree*) (root & ~ureg(1)); + TURF_ASSERT(m_flatTree->getSize() > 0); + m_flatTreeIdx = 0; + m_table = m_flatTree->getTables()[0].load(turf::Consume); + TURF_ASSERT(m_table); + m_idx = -1; + } else { + m_flatTree = NULL; + m_flatTreeIdx = 0; + m_table = (typename Details::Table*) root; + m_idx = -1; + } + if (m_table) { + next(); + } else { + m_hash = KeyTraits::NullHash; + m_value = Value(ValueTraits::NullValue); + } + } + + void next() { + TURF_ASSERT(m_table); + TURF_ASSERT(isValid() || m_idx == -1); // Either the Iterator is already valid, or we've just started iterating. + for (;;) { + searchInTable: + m_idx++; + if (m_idx <= m_table->sizeMask) { + // Index still inside range of table. + typename Details::CellGroup* group = m_table->getCellGroups() + (m_idx >> 2); + typename Details::Cell *cell = group->cells + (m_idx & 3); + m_hash = cell->hash.load(turf::Relaxed); + if (m_hash != KeyTraits::NullHash) { + // Cell has been reserved. + m_value = cell->value.load(turf::Relaxed); + TURF_ASSERT(m_value != Value(ValueTraits::Redirect)); + if (m_value != Value(ValueTraits::NullValue)) + return; // Yield this cell. + } + } else { + // We've advanced past the end of this table. + if (m_flatTree) { + // Scan for the next unique table in the flattree. + while (++m_flatTreeIdx < m_flatTree->getSize()) { + typename Details::Table* nextTable = m_flatTree->getTables()[m_flatTreeIdx].load(turf::Consume); + if (nextTable != m_table) { + // Found the next table. + m_table = nextTable; + m_idx = -1; + goto searchInTable; // Continue iterating in this table. + } + } + } + // That's the end of the entire map. + m_hash = KeyTraits::NullHash; + m_value = Value(ValueTraits::NullValue); + return; + } + } + } + + bool isValid() const { + return m_value != Value(ValueTraits::NullValue); + } + + Key getKey() const { + TURF_ASSERT(isValid()); + // Since we've forbidden concurrent inserts (for now), nonatomic would suffice here, but let's plan ahead: + return KeyTraits::dehash(m_hash); + } + + Value getValue() const { + TURF_ASSERT(isValid()); + return m_value; + } + }; +}; + +} // namespace junction + +#endif // JUNCTION_CONCURRENTMAP_GRAMPA_H diff --git a/junction/ConcurrentMap_LeapFrog.cpp b/junction/ConcurrentMap_LeapFrog.cpp new file mode 100644 index 0000000..4f1a539 --- /dev/null +++ b/junction/ConcurrentMap_LeapFrog.cpp @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include + +namespace junction { + +TURF_TRACE_DEFINE_BEGIN(ConcurrentMap_LeapFrog, 17) // autogenerated by TidySource.py +TURF_TRACE_DEFINE("[Mutator] find constructor called") +TURF_TRACE_DEFINE("[Mutator] find was redirected") +TURF_TRACE_DEFINE("[Mutator] insert constructor called") +TURF_TRACE_DEFINE("[Mutator] insert was redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] called") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] exchanged Value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] detected race to write value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] racing write inserted new value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] was redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] was re-redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] overflow after redirect") +TURF_TRACE_DEFINE("[Mutator::eraseValue] called") +TURF_TRACE_DEFINE("[Mutator::eraseValue] detected race to write value") +TURF_TRACE_DEFINE("[Mutator::eraseValue] was redirected") +TURF_TRACE_DEFINE("[Mutator::eraseValue] was re-redirected") +TURF_TRACE_DEFINE("[get] called") +TURF_TRACE_DEFINE("[get] was redirected") +TURF_TRACE_DEFINE_END(ConcurrentMap_LeapFrog, 17) + +} // namespace junction diff --git a/junction/ConcurrentMap_LeapFrog.h b/junction/ConcurrentMap_LeapFrog.h new file mode 100644 index 0000000..3cefc6f --- /dev/null +++ b/junction/ConcurrentMap_LeapFrog.h @@ -0,0 +1,336 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_CONCURRENTMAP_LEAPFROG_H +#define JUNCTION_CONCURRENTMAP_LEAPFROG_H + +#include +#include +#include +#include +#include + +namespace junction { + +TURF_TRACE_DECLARE(ConcurrentMap_LeapFrog, 17) + +template , class VT = DefaultValueTraits> +class ConcurrentMap_LeapFrog { +public: + typedef K Key; + typedef V Value; + typedef KT KeyTraits; + typedef VT ValueTraits; + typedef typename turf::util::BestFit::Unsigned Hash; + typedef details::LeapFrog Details; + +private: + turf::Atomic m_root; + +public: + ConcurrentMap_LeapFrog(ureg capacity) : m_root(Details::Table::create(capacity)) { + } + + ~ConcurrentMap_LeapFrog() { + typename Details::Table* table = m_root.loadNonatomic(); + table->destroy(); + } + + // publishTableMigration() is called by exactly one thread from Details::TableMigration::run() + // after all the threads participating in the migration have completed their work. + void publishTableMigration(typename Details::TableMigration* migration) { + // There are no racing calls to this function. + typename Details::Table* oldRoot = m_root.loadNonatomic(); + m_root.store(migration->m_destination, turf::Release); + TURF_ASSERT(oldRoot == migration->getSources()[0].table); + // Caller will GC the TableMigration and the source table. + } + + // A Mutator represents a known cell in the hash table. + // It's meant for manipulations within a temporary function scope. + // Obviously you must not call QSBR::Update while holding a Mutator. + // Any operation that modifies the table (exchangeValue, eraseValue) + // may be forced to follow a redirected cell, which changes the Mutator itself. + // Note that even if the Mutator was constructed from an existing cell, + // exchangeValue() can still trigger a resize if the existing cell was previously marked deleted, + // or if another thread deletes the key between the two steps. + class Mutator { + private: + friend class ConcurrentMap_LeapFrog; + + ConcurrentMap_LeapFrog& m_map; + typename Details::Table* m_table; + typename Details::Cell* m_cell; + Value m_value; + + // Constructor: Find existing cell + Mutator(ConcurrentMap_LeapFrog& map, Key key, bool) : m_map(map), m_value(Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_LeapFrog, 0, "[Mutator] find constructor called", uptr(m_table), uptr(key)); + Hash hash = KeyTraits::hash(key); + for (;;) { + m_table = m_map.m_root.load(turf::Consume); + m_cell = Details::find(hash, m_table); + if (!m_cell) + return; + m_value = m_cell->value.load(turf::Consume); + if (m_value != Value(ValueTraits::Redirect)) + return; // Found an existing value + // We've encountered a Redirect value. Help finish the migration. + TURF_TRACE(ConcurrentMap_LeapFrog, 1, "[Mutator] find was redirected", uptr(m_table), 0); + m_table->jobCoordinator.participate(); + // Try again using the latest root. + } + } + + // Constructor: Insert cell + Mutator(ConcurrentMap_LeapFrog& map, Key key) : m_map(map), m_table(map.m_root.load(turf::Consume)), m_value(Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_LeapFrog, 2, "[Mutator] insert constructor called", uptr(m_table), uptr(key)); + Hash hash = KeyTraits::hash(key); + for (;;) { + m_table = m_map.m_root.load(turf::Consume); + ureg overflowIdx; + switch (Details::insert(hash, m_table, m_cell, overflowIdx)) { // Modifies m_cell + case Details::InsertResult_InsertedNew: { + // We've inserted a new cell. Don't load m_cell->value. + return; + } + case Details::InsertResult_AlreadyFound: { + // The hash was already found in the table. + m_value = m_cell->value.load(turf::Consume); + if (m_value == Value(ValueTraits::Redirect)) { + // We've encountered a Redirect value. + TURF_TRACE(ConcurrentMap_LeapFrog, 3, "[Mutator] insert was redirected", uptr(m_table), uptr(m_value)); + break; // Help finish the migration. + } + return; // Found an existing value + } + case Details::InsertResult_Overflow: { + Details::beginTableMigration(m_map, m_table, overflowIdx); + break; + } + } + // A migration has been started (either by us, or another thread). Participate until it's complete. + m_table->jobCoordinator.participate(); + // Try again using the latest root. + } + } + + public: + Value getValue() const { + // Return previously loaded value. Don't load it again. + return Value(m_value); + } + + Value exchangeValue(Value desired) { + TURF_ASSERT(desired != Value(ValueTraits::NullValue)); + TURF_ASSERT(m_cell); // Cell must have been found or inserted + TURF_TRACE(ConcurrentMap_LeapFrog, 4, "[Mutator::exchangeValue] called", uptr(m_table), uptr(m_value)); + for (;;) { + Value oldValue = m_value; + if (m_cell->value.compareExchangeStrong(m_value, desired, turf::ConsumeRelease)) { + // Exchange was successful. Return previous value. + TURF_TRACE(ConcurrentMap_LeapFrog, 5, "[Mutator::exchangeValue] exchanged Value", uptr(m_value), uptr(desired)); + Value result = m_value; + m_value = desired; // Leave the mutator in a valid state + return result; + } + // The CAS failed and m_value has been updated with the latest value. + if (m_value != Value(ValueTraits::Redirect)) { + TURF_TRACE(ConcurrentMap_LeapFrog, 6, "[Mutator::exchangeValue] detected race to write value", uptr(m_table), uptr(m_value)); + if (oldValue == Value(ValueTraits::NullValue) && m_value != Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_LeapFrog, 7, "[Mutator::exchangeValue] racing write inserted new value", uptr(m_table), uptr(m_value)); + } + // There was a racing write (or erase) to this cell. + // Pretend we exchanged with ourselves, and just let the racing write win. + return desired; + } + // We've encountered a Redirect value. Help finish the migration. + TURF_TRACE(ConcurrentMap_LeapFrog, 8, "[Mutator::exchangeValue] was redirected", uptr(m_table), uptr(m_value)); + Hash hash = m_cell->hash.load(turf::Relaxed); + for (;;) { + // Help complete the migration. + m_table->jobCoordinator.participate(); + // Try again in the new table. + m_table = m_map.m_root.load(turf::Consume); + m_value = Value(ValueTraits::NullValue); + ureg overflowIdx; + switch (Details::insert(hash, m_table, m_cell, overflowIdx)) { // Modifies m_cell + case Details::InsertResult_AlreadyFound: + m_value = m_cell->value.load(turf::Consume); + if (m_value == Value(ValueTraits::Redirect)) { + TURF_TRACE(ConcurrentMap_LeapFrog, 9, "[Mutator::exchangeValue] was re-redirected", uptr(m_table), uptr(m_value)); + break; + } + goto breakOuter; + case Details::InsertResult_InsertedNew: + goto breakOuter; + case Details::InsertResult_Overflow: + TURF_TRACE(ConcurrentMap_LeapFrog, 10, "[Mutator::exchangeValue] overflow after redirect", uptr(m_table), overflowIdx); + Details::beginTableMigration(m_map, m_table, overflowIdx); + break; + } + // We were redirected... again + } + breakOuter: + ; + // Try again in the new table. + } + } + + void setValue(Value desired) { + exchangeValue(desired); + } + + Value eraseValue() { + TURF_ASSERT(m_cell); // Cell must have been found or inserted + TURF_TRACE(ConcurrentMap_LeapFrog, 11, "[Mutator::eraseValue] called", uptr(m_table), uptr(m_cell)); + for (;;) { + if (m_value == Value(ValueTraits::NullValue)) + return Value(m_value); + TURF_ASSERT(m_cell); // m_value is non-NullValue, therefore cell must have been found or inserted. + if (m_cell->value.compareExchangeStrong(m_value, Value(ValueTraits::NullValue), turf::Consume)) { + // Exchange was successful and a non-NULL value was erased and returned by reference in m_value. + TURF_ASSERT(m_value != ValueTraits::NullValue); // Implied by the test at the start of the loop. + Value result = m_value; + m_value = Value(ValueTraits::NullValue); // Leave the mutator in a valid state + return result; + } + // The CAS failed and m_value has been updated with the latest value. + TURF_TRACE(ConcurrentMap_LeapFrog, 12, "[Mutator::eraseValue] detected race to write value", uptr(m_table), uptr(m_cell)); + if (m_value != Value(ValueTraits::Redirect)) { + // There was a racing write (or erase) to this cell. + // Pretend we erased nothing, and just let the racing write win. + return Value(ValueTraits::NullValue); + } + // We've been redirected to a new table. + TURF_TRACE(ConcurrentMap_LeapFrog, 13, "[Mutator::eraseValue] was redirected", uptr(m_table), uptr(m_cell)); + Hash hash = m_cell->hash.load(turf::Relaxed); // Re-fetch hash + for (;;) { + // Help complete the migration. + m_table->jobCoordinator.participate(); + // Try again in the new table. + m_table = m_map.m_root.load(turf::Consume); + m_cell = Details::find(hash, m_table); + if (!m_cell) { + m_value = Value(ValueTraits::NullValue); + return m_value; + } + m_value = m_cell->value.load(turf::Relaxed); + if (m_value != Value(ValueTraits::Redirect)) + break; + TURF_TRACE(ConcurrentMap_LeapFrog, 14, "[Mutator::eraseValue] was re-redirected", uptr(m_table), uptr(m_cell)); + } + } + } + }; + + Mutator insert(Key key) { + return Mutator(*this, key); + } + + Mutator find(Key key) { + return Mutator(*this, key, false); + } + + // Lookup without creating a temporary Mutator. + Value get(Key key) { + Hash hash = KeyTraits::hash(key); + TURF_TRACE(ConcurrentMap_LeapFrog, 15, "[get] called", uptr(hash), 0); + for (;;) { + typename Details::Table* table = m_root.load(turf::Consume); + typename Details::Cell* cell = Details::find(hash, table); + if (!cell) + return Value(ValueTraits::NullValue); + Value value = cell->value.load(turf::Consume); + if (value != Value(ValueTraits::Redirect)) + return value; // Found an existing value + // We've been redirected to a new table. Help with the migration. + TURF_TRACE(ConcurrentMap_LeapFrog, 16, "[get] was redirected", uptr(table), uptr(hash)); + table->jobCoordinator.participate(); + // Try again in the new table. + } + } + + Value insert(Key key, Value desired) { + Mutator iter(*this, key); + return iter.exchangeValue(desired); + } + + Value exchange(Key key, Value desired) { + Mutator iter(*this, key); + return iter.exchangeValue(desired); + } + + Value erase(Key key) { + Mutator iter(*this, key, false); + return iter.eraseValue(); + } + + // The easiest way to implement an Iterator is to prevent all Redirects. + // The currrent Iterator does that by forbidding concurrent inserts. + // To make it work with concurrent inserts, we'd need a way to block TableMigrations. + class Iterator { + private: + typename Details::Table* m_table; + ureg m_idx; + Key m_hash; + Value m_value; + + public: + Iterator(ConcurrentMap_LeapFrog& map) { + // Since we've forbidden concurrent inserts (for now), nonatomic would suffice here, but let's plan ahead: + m_table = map.m_root.load(turf::Consume); + m_idx = -1; + next(); + } + + void next() { + TURF_ASSERT(m_table); + TURF_ASSERT(isValid() || m_idx == -1); // Either the Iterator is already valid, or we've just started iterating. + while (++m_idx <= m_table->sizeMask) { + // Index still inside range of table. + typename Details::CellGroup* group = m_table->getCellGroups() + (m_idx >> 2); + typename Details::Cell *cell = group->cells + (m_idx & 3); + m_hash = cell->hash.load(turf::Relaxed); + if (m_hash != KeyTraits::NullHash) { + // Cell has been reserved. + m_value = cell->value.load(turf::Relaxed); + TURF_ASSERT(m_value != Value(ValueTraits::Redirect)); + if (m_value != Value(ValueTraits::NullValue)) + return; // Yield this cell. + } + } + // That's the end of the map. + m_hash = KeyTraits::NullHash; + m_value = ValueTraits::NullValue; + } + + bool isValid() const { + return m_value != Value(ValueTraits::NullValue); + } + + Key getKey() const { + TURF_ASSERT(isValid()); + // Since we've forbidden concurrent inserts (for now), nonatomic would suffice here, but let's plan ahead: + return KeyTraits::dehash(m_hash); + } + + Value getValue() const { + TURF_ASSERT(isValid()); + return m_value; + } + }; +}; + +} // namespace junction + +#endif // JUNCTION_CONCURRENTMAP_LEAPFROG_H diff --git a/junction/ConcurrentMap_Linear.cpp b/junction/ConcurrentMap_Linear.cpp new file mode 100644 index 0000000..f351594 --- /dev/null +++ b/junction/ConcurrentMap_Linear.cpp @@ -0,0 +1,38 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include + +namespace junction { + +TURF_TRACE_DEFINE_BEGIN(ConcurrentMap_Linear, 18) // autogenerated by TidySource.py +TURF_TRACE_DEFINE("[Mutator] find constructor called") +TURF_TRACE_DEFINE("[Mutator] find was redirected") +TURF_TRACE_DEFINE("[Mutator] insert constructor called") +TURF_TRACE_DEFINE("[Mutator] insert was redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] called") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] ran out of valuesRemaining") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] exchanged Value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] detected race to write value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] racing write inserted new value") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] was redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] was re-redirected") +TURF_TRACE_DEFINE("[Mutator::exchangeValue] overflow after redirect") +TURF_TRACE_DEFINE("[Mutator::eraseValue] called") +TURF_TRACE_DEFINE("[Mutator::eraseValue] detected race to write value") +TURF_TRACE_DEFINE("[Mutator::eraseValue] was redirected") +TURF_TRACE_DEFINE("[Mutator::eraseValue] was re-redirected") +TURF_TRACE_DEFINE("[get] called") +TURF_TRACE_DEFINE("[get] was redirected") +TURF_TRACE_DEFINE_END(ConcurrentMap_Linear, 18) + +} // namespace junction diff --git a/junction/ConcurrentMap_Linear.h b/junction/ConcurrentMap_Linear.h new file mode 100644 index 0000000..1de64b2 --- /dev/null +++ b/junction/ConcurrentMap_Linear.h @@ -0,0 +1,358 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_CONCURRENTMAP_LINEAR_H +#define JUNCTION_CONCURRENTMAP_LINEAR_H + +#include +#include +#include +#include +#include + +namespace junction { + +TURF_TRACE_DECLARE(ConcurrentMap_Linear, 18) + +template , class VT = DefaultValueTraits> +class ConcurrentMap_Linear { +public: + typedef K Key; + typedef V Value; + typedef KT KeyTraits; + typedef VT ValueTraits; + typedef typename turf::util::BestFit::Unsigned Hash; + typedef details::Linear Details; + +private: + turf::Atomic m_root; + +public: + ConcurrentMap_Linear(ureg capacity) { + ureg limitNumValues = capacity * 3 / 4; + m_root.storeNonatomic(Details::Table::create(capacity, limitNumValues)); + } + + ~ConcurrentMap_Linear() { + typename Details::Table* table = m_root.loadNonatomic(); + table->destroy(); + } + + // publishTableMigration() is called by exactly one thread from Details::TableMigration::run() + // after all the threads participating in the migration have completed their work. + void publishTableMigration(typename Details::TableMigration* migration) { + // There are no racing calls to this function. + typename Details::Table* oldRoot = m_root.loadNonatomic(); + m_root.store(migration->m_destination, turf::Release); + TURF_ASSERT(oldRoot == migration->m_source); + // Caller will GC the TableMigration and the source table. + } + + // A Mutator represents a known cell in the hash table. + // It's meant for manipulations within a temporary function scope. + // Obviously you must not call QSBR::Update while holding a Mutator. + // Any operation that modifies the table (exchangeValue, eraseValue) + // may be forced to follow a redirected cell, which changes the Mutator itself. + // Note that even if the Mutator was constructed from an existing cell, + // exchangeValue() can still trigger a resize if the existing cell was previously marked deleted, + // or if another thread deletes the key between the two steps. + class Mutator { + private: + friend class ConcurrentMap_Linear; + + ConcurrentMap_Linear& m_map; + typename Details::Table* m_table; + typename Details::Cell* m_cell; + Value m_value; + + // Constructor: Find existing cell + Mutator(ConcurrentMap_Linear& map, Key key, bool) : m_map(map), m_value(Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_Linear, 0, "[Mutator] find constructor called", uptr(m_table), uptr(key)); + Hash hash = KeyTraits::hash(key); + for (;;) { + m_table = m_map.m_root.load(turf::Consume); + m_cell = Details::find(hash, m_table); + if (!m_cell) + return; + m_value = m_cell->value.load(turf::Consume); + if (m_value != Value(ValueTraits::Redirect)) + return; // Found an existing value + // We've encountered a Redirect value. Help finish the migration. + TURF_TRACE(ConcurrentMap_Linear, 1, "[Mutator] find was redirected", uptr(m_table), uptr(0)); + m_table->jobCoordinator.participate(); + // Try again using the latest root. + } + } + + // Constructor: Insert cell + Mutator(ConcurrentMap_Linear& map, Key key) : m_map(map), m_table(map.m_root.load(turf::Consume)), m_value(Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_Linear, 2, "[Mutator] insert constructor called", uptr(m_table), uptr(key)); + Hash hash = KeyTraits::hash(key); + for (;;) { + m_table = m_map.m_root.load(turf::Consume); + switch (Details::insert(hash, m_table, m_cell)) { // Modifies m_cell + case Details::InsertResult_InsertedNew: { + // We've inserted a new cell. Don't load m_cell->value. + return; + } + case Details::InsertResult_AlreadyFound: { + // The hash was already found in the table. + m_value = m_cell->value.load(turf::Consume); + if (m_value == Value(ValueTraits::Redirect)) { + // We've encountered a Redirect value. + TURF_TRACE(ConcurrentMap_Linear, 3, "[Mutator] insert was redirected", uptr(m_table), uptr(m_value)); + break; // Help finish the migration. + } + return; // Found an existing value + } + case Details::InsertResult_Overflow: { + Details::beginTableMigration(m_map, m_table); + break; + } + } + // A migration has been started (either by us, or another thread). Participate until it's complete. + m_table->jobCoordinator.participate(); + // Try again using the latest root. + } + } + + public: + Value getValue() const { + // Return previously loaded value. Don't load it again. + return Value(m_value); + } + + Value exchangeValue(Value desired) { + TURF_ASSERT(desired != Value(ValueTraits::NullValue)); + TURF_ASSERT(m_cell); // Cell must have been found or inserted + TURF_TRACE(ConcurrentMap_Linear, 4, "[Mutator::exchangeValue] called", uptr(m_table), uptr(m_value)); + for (;;) { + Value oldValue = m_value; + s32 prevRemainingValues = s32(-1); + if (oldValue == Value(ValueTraits::NullValue)) { + // It's a deleted or newly initialized cell. + // Decrement remainingValues to ensure we have permission to (re)insert a value. + prevRemainingValues = m_table->valuesRemaining.fetchSub(1, turf::Relaxed); + if (prevRemainingValues <= 0) { + TURF_TRACE(ConcurrentMap_Linear, 5, "[Mutator::exchangeValue] ran out of valuesRemaining", uptr(m_table), prevRemainingValues); + // Can't (re)insert any more values. + // There are two ways this can happen. One with a TableMigration already in progress, and one without: + // 1. A TableMigration puts a cap on the number of values late-arriving threads are allowed to insert. + // 2. Two threads race to insert the same key, and it's the last free cell in the table. + // (Note: We could get tid of the valuesRemaining counter by handling the possibility of migration failure, + // as LeapFrog and Grampa do...) + m_table->valuesRemaining.fetchAdd(1, turf::Relaxed); // Undo valuesRemaining decrement + // Since we don't know whether there's already a TableMigration in progress, always attempt to start one here: + Details::beginTableMigration(m_map, m_table); + goto helpMigrate; + } + } + if (m_cell->value.compareExchangeStrong(m_value, desired, turf::ConsumeRelease)) { + // Exchange was successful. Return previous value. + TURF_TRACE(ConcurrentMap_Linear, 6, "[Mutator::exchangeValue] exchanged Value", uptr(m_value), uptr(desired)); + Value result = m_value; + m_value = desired; // Leave the mutator in a valid state + return result; + } + // The CAS failed and m_value has been updated with the latest value. + if (m_value != Value(ValueTraits::Redirect)) { + TURF_TRACE(ConcurrentMap_Linear, 7, "[Mutator::exchangeValue] detected race to write value", uptr(m_table), uptr(m_value)); + if (oldValue == Value(ValueTraits::NullValue) && m_value != Value(ValueTraits::NullValue)) { + TURF_TRACE(ConcurrentMap_Linear, 8, "[Mutator::exchangeValue] racing write inserted new value", uptr(m_table), uptr(m_value)); + m_table->valuesRemaining.fetchAdd(1, turf::Relaxed); // Undo valuesRemaining decrement + } + // There was a racing write (or erase) to this cell. + // Pretend we exchanged with ourselves, and just let the racing write win. + return desired; + } + // We've encountered a Redirect value. Help finish the migration. + TURF_TRACE(ConcurrentMap_Linear, 9, "[Mutator::exchangeValue] was redirected", uptr(m_table), uptr(m_value)); + helpMigrate: + // Help migrate to new table. + Hash hash = m_cell->hash.load(turf::Relaxed); + for (;;) { + // Help complete the migration. + m_table->jobCoordinator.participate(); + // Try again in the new table. + m_table = m_map.m_root.load(turf::Consume); + m_value = Value(ValueTraits::NullValue); + switch (Details::insert(hash, m_table, m_cell)) { // Modifies m_cell + case Details::InsertResult_AlreadyFound: + m_value = m_cell->value.load(turf::Consume); + if (m_value == Value(ValueTraits::Redirect)) { + TURF_TRACE(ConcurrentMap_Linear, 10, "[Mutator::exchangeValue] was re-redirected", uptr(m_table), uptr(m_value)); + break; + } + goto breakOuter; + case Details::InsertResult_InsertedNew: + goto breakOuter; + case Details::InsertResult_Overflow: + TURF_TRACE(ConcurrentMap_Linear, 11, "[Mutator::exchangeValue] overflow after redirect", uptr(m_table), 0); + Details::beginTableMigration(m_map, m_table); + break; + } + // We were redirected... again + } + breakOuter: + ; + // Try again in the new table. + } + } + + void setValue(Value desired) { + exchangeValue(desired); + } + + Value eraseValue() { + TURF_ASSERT(m_cell); // Cell must have been found or inserted + TURF_TRACE(ConcurrentMap_Linear, 12, "[Mutator::eraseValue] called", uptr(m_table), m_cell - m_table->getCells()); + for (;;) { + if (m_value == Value(ValueTraits::NullValue)) + return Value(m_value); + TURF_ASSERT(m_cell); // m_value is non-NullValue, therefore cell must have been found or inserted. + if (m_cell->value.compareExchangeStrong(m_value, Value(ValueTraits::NullValue), turf::Consume)) { + // Exchange was successful and a non-NULL value was erased and returned by reference in m_value. + TURF_ASSERT(m_value != ValueTraits::NullValue); // Implied by the test at the start of the loop. + m_table->valuesRemaining.fetchAdd(1, turf::Relaxed); + Value result = m_value; + m_value = Value(ValueTraits::NullValue); // Leave the mutator in a valid state + return result; + } + // The CAS failed and m_value has been updated with the latest value. + TURF_TRACE(ConcurrentMap_Linear, 13, "[Mutator::eraseValue] detected race to write value", uptr(m_table), m_cell - m_table->getCells()); + if (m_value != Value(ValueTraits::Redirect)) { + // There was a racing write (or erase) to this cell. + // Pretend we erased nothing, and just let the racing write win. + return Value(ValueTraits::NullValue); + } + // We've been redirected to a new table. + TURF_TRACE(ConcurrentMap_Linear, 14, "[Mutator::eraseValue] was redirected", uptr(m_table), m_cell - m_table->getCells()); + Hash hash = m_cell->hash.load(turf::Relaxed); // Re-fetch hash + for (;;) { + // Help complete the migration. + m_table->jobCoordinator.participate(); + // Try again in the new table. + m_table = m_map.m_root.load(turf::Consume); + m_cell = Details::find(hash, m_table); + if (!m_cell) { + m_value = Value(ValueTraits::NullValue); + return m_value; + } + m_value = m_cell->value.load(turf::Relaxed); + if (m_value != Value(ValueTraits::Redirect)) + break; + TURF_TRACE(ConcurrentMap_Linear, 15, "[Mutator::eraseValue] was re-redirected", uptr(m_table), m_cell - m_table->getCells()); + } + } + } + }; + + Mutator insert(Key key) { + return Mutator(*this, key); + } + + Mutator find(Key key) { + return Mutator(*this, key, false); + } + + // Lookup without creating a temporary Mutator. + Value get(Key key) { + Hash hash = KeyTraits::hash(key); + TURF_TRACE(ConcurrentMap_Linear, 16, "[get] called", uptr(this), uptr(hash)); + for (;;) { + typename Details::Table* table = m_root.load(turf::Consume); + typename Details::Cell* cell = Details::find(hash, table); + if (!cell) + return Value(ValueTraits::NullValue); + Value value = cell->value.load(turf::Consume); + if (value != Value(ValueTraits::Redirect)) + return value; // Found an existing value + // We've been redirected to a new table. Help with the migration. + TURF_TRACE(ConcurrentMap_Linear, 17, "[get] was redirected", uptr(table), uptr(cell)); + table->jobCoordinator.participate(); + // Try again in the new table. + } + } + + Value insert(Key key, Value desired) { + Mutator iter(*this, key); + return iter.exchangeValue(desired); + } + + Value exchange(Key key, Value desired) { + Mutator iter(*this, key); + return iter.exchangeValue(desired); + } + + Value erase(Key key) { + Mutator iter(*this, key, false); + return iter.eraseValue(); + } + + // The easiest way to implement an Iterator is to prevent all Redirects. + // The currrent Iterator does that by forbidding concurrent inserts. + // To make it work with concurrent inserts, we'd need a way to block TableMigrations. + class Iterator { + private: + typename Details::Table* m_table; + ureg m_idx; + Key m_hash; + Value m_value; + + public: + Iterator(ConcurrentMap_Linear& map) { + // Since we've forbidden concurrent inserts (for now), nonatomic would suffice here, but let's plan ahead: + m_table = map.m_root.load(turf::Consume); + m_idx = -1; + next(); + } + + void next() { + TURF_ASSERT(m_table); + TURF_ASSERT(isValid() || m_idx == -1); // Either the Iterator is already valid, or we've just started iterating. + while (++m_idx <= m_table->sizeMask) { + // Index still inside range of table. + typename Details::Cell *cell = m_table->getCells() + m_idx; + m_hash = cell->hash.load(turf::Relaxed); + if (m_hash != KeyTraits::NullHash) { + // Cell has been reserved. + m_value = cell->value.load(turf::Relaxed); + TURF_ASSERT(m_value != Value(ValueTraits::Redirect)); + if (m_value != Value(ValueTraits::NullValue)) + return; // Yield this cell. + } + } + // That's the end of the map. + m_hash = KeyTraits::NullHash; + m_value = ValueTraits::NullValue; + } + + bool isValid() const { + return m_value != Value(ValueTraits::NullValue); + } + + Key getKey() const { + TURF_ASSERT(isValid()); + // Since we've forbidden concurrent inserts (for now), nonatomic would suffice here, but let's plan ahead: + return KeyTraits::dehash(m_hash); + } + + Value getValue() const { + TURF_ASSERT(isValid()); + return m_value; + } + }; +}; + +} // namespace junction + +#endif // JUNCTION_CONCURRENTMAP_LINEAR_H diff --git a/junction/Core.h b/junction/Core.h new file mode 100644 index 0000000..aacc8a7 --- /dev/null +++ b/junction/Core.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_CORE_H +#define JUNCTION_CORE_H + +//----------------------------------------------- +#include "junction_config.h" // junction_config.h generated by CMake. + +// Default to true in case junction_config.h is missing entirely: +#ifndef JUNCTION_USE_STRIPING +#define JUNCTION_USE_STRIPING 1 +#endif +//----------------------------------------------- + +#include +#include +#include + +namespace junction { +using namespace turf::intTypes; +} + +#endif // JUNCTION_CORE_H diff --git a/junction/MapTraits.h b/junction/MapTraits.h new file mode 100644 index 0000000..de83d27 --- /dev/null +++ b/junction/MapTraits.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_MAPTRAITS_H +#define JUNCTION_MAPTRAITS_H + +#include +#include + +namespace junction { + +template +struct DefaultKeyTraits { + typedef T Key; + typedef typename turf::util::BestFit::Unsigned Hash; + static const Hash NullHash = Hash(0); + static Hash hash(T key) { + return turf::util::avalanche(Hash(key)); + } + static Key dehash(Hash hash) { + return (T) turf::util::deavalanche(hash); + } +}; + +template +struct DefaultValueTraits { + typedef T Value; + typedef typename turf::util::BestFit::Unsigned IntType; + static const IntType NullValue = 0; + static const IntType Redirect = 1; +}; + +} // namespace junction + +#endif // JUNCTION_MAPTRAITS_H diff --git a/junction/QSBR.cpp b/junction/QSBR.cpp new file mode 100644 index 0000000..eaab358 --- /dev/null +++ b/junction/QSBR.cpp @@ -0,0 +1,107 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include +#include +#include +#include + +namespace junction { + +QSBR DefaultQSBR; + +QSBR::Context QSBR::createContext() { + turf::LockGuard guard(m_mutex); + TURF_RACE_DETECT_GUARD(m_flushRaceDetector); + m_numContexts++; + m_remaining++; + TURF_ASSERT(m_numContexts < (1 << 14)); + sreg context = m_freeIndex; + if (context >= 0) { + TURF_ASSERT(context < (sreg) m_status.size()); + TURF_ASSERT(!m_status[context].inUse); + m_freeIndex = m_status[context].nextFree; + m_status[context] = Status(); + } else { + context = m_status.size(); + m_status.push_back(Status()); + } + return context; +} + +void QSBR::destroyContext(QSBR::Context context) { + std::vector actions; + { + turf::LockGuard guard(m_mutex); + TURF_RACE_DETECT_GUARD(m_flushRaceDetector); + TURF_ASSERT(context < m_status.size()); + if (m_status[context].inUse && !m_status[context].wasIdle) { + TURF_ASSERT(m_remaining > 0); + --m_remaining; + } + m_status[context].inUse = 0; + m_status[context].nextFree = m_freeIndex; + m_freeIndex = context; + m_numContexts--; + if (m_remaining == 0) + onAllQuiescentStatesPassed(actions); + } + for (ureg i = 0; i < actions.size(); i++) + actions[i](); +} + +void QSBR::onAllQuiescentStatesPassed(std::vector& actions) { + // m_mutex must be held + actions.swap(m_pendingActions); + m_pendingActions.swap(m_deferredActions); + m_remaining = m_numContexts; + for (ureg i = 0; i < m_status.size(); i++) + m_status[i].wasIdle = 0; +} + +void QSBR::update(QSBR::Context context) { + std::vector actions; + { + turf::LockGuard guard(m_mutex); + TURF_RACE_DETECT_GUARD(m_flushRaceDetector); + TURF_ASSERT(context < m_status.size()); + Status& status = m_status[context]; + TURF_ASSERT(status.inUse); + if (status.wasIdle) + return; + status.wasIdle = 1; + TURF_ASSERT(m_remaining > 0); + if (--m_remaining > 0) + return; + onAllQuiescentStatesPassed(actions); + } + for (ureg i = 0; i < actions.size(); i++) + actions[i](); +} + +void QSBR::flush() { + // This is like saying that all contexts are quiescent, + // so we can issue all actions at once. + // No lock is taken. + TURF_RACE_DETECT_GUARD(m_flushRaceDetector); // There should be no concurrent operations + for (ureg i = 0; i < m_pendingActions.size(); i++) + m_pendingActions[i](); + m_pendingActions.clear(); + for (ureg i = 0; i < m_deferredActions.size(); i++) + m_deferredActions[i](); + m_deferredActions.clear(); + m_remaining = m_numContexts; +} + +} // namespace junction diff --git a/junction/QSBR.h b/junction/QSBR.h new file mode 100644 index 0000000..98a9a1b --- /dev/null +++ b/junction/QSBR.h @@ -0,0 +1,91 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_QSBR_H +#define JUNCTION_QSBR_H + +#include +#include +#include +#include +#include + +namespace junction { + +class QSBR { +private: + struct Action { + void (*func)(void*); + uptr param[4]; // Size limit found experimentally. Verified by assert below. + + Action(void (*f)(void*), void* p, ureg paramSize) : func(f) { + TURF_ASSERT(paramSize <= sizeof(param)); // Verify size limit. + memcpy(¶m, p, paramSize); + } + void operator()() { + func(¶m); + } + }; + + struct Status { + u16 inUse : 1; + u16 wasIdle : 1; + s16 nextFree : 14; + + Status() : inUse(1), wasIdle(0), nextFree(0) { + } + }; + + turf::Mutex m_mutex; + TURF_DEFINE_RACE_DETECTOR(m_flushRaceDetector) + std::vector m_status; + sreg m_freeIndex; + sreg m_numContexts; + sreg m_remaining; + std::vector m_deferredActions; + std::vector m_pendingActions; + + void onAllQuiescentStatesPassed(std::vector& callbacks); + +public: + typedef u16 Context; + + QSBR() : m_freeIndex(-1), m_numContexts(0), m_remaining(0) { + } + Context createContext(); + void destroyContext(Context context); + + template + void enqueue(void (T::*pmf)(), T* target) { + struct Closure { + void (T::*pmf)(); + T* target; + static void thunk(void* param) { + Closure* self = (Closure*) param; + TURF_CALL_MEMBER(*self->target, self->pmf)(); + } + }; + Closure closure = { pmf, target }; + turf::LockGuard guard(m_mutex); + TURF_RACE_DETECT_GUARD(m_flushRaceDetector); + m_deferredActions.push_back(Action(Closure::thunk, &closure, sizeof(closure))); + } + + void update(Context context); + void flush(); +}; + +extern QSBR DefaultQSBR; + +} // junction + +#endif // JUNCTION_QSBR_H diff --git a/junction/SimpleJobCoordinator.h b/junction/SimpleJobCoordinator.h new file mode 100644 index 0000000..2732497 --- /dev/null +++ b/junction/SimpleJobCoordinator.h @@ -0,0 +1,92 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_SIMPLEJOBCOORDINATOR_H +#define JUNCTION_SIMPLEJOBCOORDINATOR_H + +#include +#include + +namespace junction { + +// It's safe to call everything here from within a Job itself. +// In particular, you're allowed to particpate() recursively. +// We actually do this in ConcurrentMap_Grampa::publish() when migrating a new flattree. +class SimpleJobCoordinator { +public: + struct Job { + virtual ~Job() { + } + virtual void run() = 0; + }; + +private: + JUNCTION_STRIPED_CONDITIONBANK_DEFINE_MEMBER() + turf::Atomic m_job; + +public: + SimpleJobCoordinator() : m_job(uptr(NULL)) { + } + + Job* loadConsume() const { + return (Job*) m_job.load(turf::Consume); + } + + void storeRelease(Job* job) { + junction::striped::ConditionPair& pair = JUNCTION_STRIPED_CONDITIONBANK_GET(this); + { + turf::LockGuard guard(pair.mutex); + m_job.store(uptr(job), turf::Release); + } + pair.condVar.wakeAll(); + } + + void participate() { + junction::striped::ConditionPair& pair = JUNCTION_STRIPED_CONDITIONBANK_GET(this); + uptr prevJob = uptr(NULL); + for (;;) { + uptr job = m_job.load(turf::Consume); + if (job == prevJob) { + turf::LockGuard guard(pair.mutex); + for (;;) { + job = m_job.loadNonatomic(); // No concurrent writes inside lock + if (job != prevJob) + break; + pair.condVar.wait(guard); + } + } + if (job == 1) + return; + reinterpret_cast(job)->run(); + prevJob = job; + } + } + + void runOne(Job* job) { + TURF_ASSERT(job != (Job*) m_job.load(turf::Relaxed)); + storeRelease(job); + job->run(); + } + + void end() { + junction::striped::ConditionPair& pair = JUNCTION_STRIPED_CONDITIONBANK_GET(this); + { + turf::LockGuard guard(pair.mutex); + m_job.store(1, turf::Release); + } + pair.condVar.wakeAll(); + } +}; + +} // namespace junction + +#endif // JUNCTION_SIMPLEJOBCOORDINATOR_H diff --git a/junction/SingleMap_Linear.h b/junction/SingleMap_Linear.h new file mode 100644 index 0000000..5ea2521 --- /dev/null +++ b/junction/SingleMap_Linear.h @@ -0,0 +1,207 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_SINGLEMAP_LINEAR_H +#define JUNCTION_SINGLEMAP_LINEAR_H + +#include +#include +#include +#include + +namespace junction { + +template , class ValueTraits = DefaultValueTraits> +class SingleMap_Linear { +private: + typedef typename KeyTraits::Hash Hash; + + struct Cell { + Hash hash; + Value value; + Cell(Hash hash, Value value) : hash(hash), value(value) { + } + }; + + Cell* m_cells; + u32 m_sizeMask; + u32 m_population; + + static Cell* createTable(ureg size) { + Cell* cells = (Cell*) TURF_HEAP.alloc(sizeof(Cell) * size); + for (ureg i = 0; i < size; i++) + new(cells + i) Cell(KeyTraits::NullHash, Value(ValueTraits::NullValue)); + return cells; + } + + static void destroyTable(Cell* cells, ureg size) { + for (ureg i = 0; i < size; i++) + cells[i].~Cell(); + TURF_HEAP.free(cells); + } + + static bool isOverpopulated(ureg population, ureg sizeMask) { + return (population * 4) >= (sizeMask * 3); + } + + void migrateToNewTable(size_t desiredSize) { + TURF_ASSERT(turf::util::isPowerOf2(desiredSize)); + Cell* srcCells = m_cells; + ureg srcSize = m_sizeMask + 1; + m_cells = createTable(desiredSize); + m_sizeMask = desiredSize - 1; + for (ureg srcIdx = 0; srcIdx < srcSize; srcIdx++) { + Cell* srcCell = srcCells + srcIdx; + if (srcCell->hash != KeyTraits::NullHash) { + for (ureg dstIdx = srcCell->hash;; dstIdx++) { + dstIdx &= m_sizeMask; + if (m_cells[dstIdx].hash == KeyTraits::NullHash) { + m_cells[dstIdx] = *srcCell; + break; + } + } + } + } + destroyTable(srcCells, srcSize); + } + +public: + SingleMap_Linear(size_t initialSize = 8) : m_cells(createTable(initialSize)), m_sizeMask(initialSize - 1), m_population(0) { + TURF_ASSERT(turf::util::isPowerOf2(initialSize)); + } + + ~SingleMap_Linear() { + destroyTable(m_cells, m_sizeMask + 1); + } + + class Iterator { + private: + friend class SingleMap_Linear; + SingleMap_Linear& m_map; + Cell* m_cell; + + // Constructor: Find without insert + Iterator(SingleMap_Linear& map, Key key, bool) : m_map(map) { + Hash hash = KeyTraits::hash(key); + TURF_ASSERT(hash != KeyTraits::NullHash); + for (ureg idx = hash;; idx++) { + idx &= map.m_sizeMask; + m_cell = map.m_cells + idx; + if (m_cell->hash == hash) + return; // Key found in table. + if (m_cell->hash != KeyTraits::NullHash) + continue; // Slot is taken by another key. Try next slot. + m_cell = NULL; // Insert not allowed & key not found. + return; + } + } + + // Constructor: Find with insert + Iterator(SingleMap_Linear& map, Key key) : m_map(map) { + Hash hash = KeyTraits::hash(key); + TURF_ASSERT(hash != KeyTraits::NullHash); + for (;;) { + for (ureg idx = hash;; idx++) { + idx &= map.m_sizeMask; + m_cell = map.m_cells + idx; + if (m_cell->hash == hash) + return; // Key found in table. + if (m_cell->hash != KeyTraits::NullHash) + continue; // Slot is taken by another key. Try next slot. + // Insert is allowed. Reserve this cell. + if (isOverpopulated(map.m_population, map.m_sizeMask)) { + map.migrateToNewTable((map.m_sizeMask + 1) * 2); + break; // Retry in new table. + } + map.m_population++; + m_cell->hash = hash; + TURF_ASSERT(m_cell->value == NULL); + return; + } + } + } + + ~Iterator() { + TURF_ASSERT(!m_cell || m_cell->value != NULL); // Forbid leaving a cell half-inserted. + } + + public: + bool isValid() const { + return !!m_cell; + } + + Value getValue() const { + TURF_ASSERT(m_cell); + return m_cell->value; + } + + Value exchangeValue(Value desired) { + TURF_ASSERT(m_cell); + TURF_ASSERT(desired != NULL); // Use eraseValue() + Value oldValue = m_cell->value; + m_cell->value = desired; + return oldValue; + } + + Value erase() { + TURF_ASSERT(m_cell); + TURF_ASSERT(m_cell->value != NULL); // Forbid erasing a cell that's not actually used. + Value oldValue = m_cell->value; + // Remove this cell by shuffling neighboring cells so there are no gaps in anyone's probe chain + ureg cellIdx = m_cell - m_map.m_cells; + for (ureg neighborIdx = cellIdx + 1;; neighborIdx++) { + neighborIdx &= m_map.m_sizeMask; + Cell* neighbor = m_map.m_cells + neighborIdx; + if (neighbor->hash == KeyTraits::NullHash) { + // Go ahead and clear this cell. It won't break anyone else's probe chain. + m_cell->hash = KeyTraits::NullHash; + m_cell->value = NULL; + m_cell = NULL; + m_map.m_population--; + return oldValue; + } + ureg idealIdx = neighbor->hash & m_map.m_sizeMask; + if (((cellIdx - idealIdx) & m_map.m_sizeMask) < ((neighborIdx - idealIdx) & m_map.m_sizeMask)) { + // Swap with neighbor, then make neighbor the new cell to remove. + *m_cell = *neighbor; + m_cell = neighbor; + cellIdx = neighborIdx; + } + } + } + }; + + Iterator insertOrFindKey(const Key& key) { + return Iterator(*this, key); + } + + Value get(const Key& key) { + Iterator iter(*this, key, false); + return iter.isValid() ? iter.getValue() : NULL; + } + + Value insert(const Key& key, Value desired) { + Iterator iter(*this, key); + return iter.exchangeValue(desired); + } + + Value erase(const Key& key) { + Iterator iter(*this, key, false); + if (iter.isValid()) + return iter.erase(); + return NULL; + } +}; + +} // namespace junction + +#endif // JUNCTION_SINGLEMAP_LINEAR_H diff --git a/junction/details/Grampa.cpp b/junction/details/Grampa.cpp new file mode 100644 index 0000000..3b33fa5 --- /dev/null +++ b/junction/details/Grampa.cpp @@ -0,0 +1,65 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include + +namespace junction { +namespace details { + +#if JUNCTION_TRACK_GRAMPA_STATS +GrampaStats GrampaStats::Instance; +#endif + +TURF_TRACE_DEFINE_BEGIN(Grampa, 37) // autogenerated by TidySource.py +TURF_TRACE_DEFINE("[find] called") +TURF_TRACE_DEFINE("[find] found existing cell optimistically") +TURF_TRACE_DEFINE("[find] found existing cell") +TURF_TRACE_DEFINE("[insert] called") +TURF_TRACE_DEFINE("[insert] reserved first cell") +TURF_TRACE_DEFINE("[insert] race to reserve first cell") +TURF_TRACE_DEFINE("[insert] found in first cell") +TURF_TRACE_DEFINE("[insert] race to read hash") +TURF_TRACE_DEFINE("[insert] found in probe chain") +TURF_TRACE_DEFINE("[insert] reserved cell") +TURF_TRACE_DEFINE("[insert] race to reserve cell") +TURF_TRACE_DEFINE("[insert] found outside probe chain") +TURF_TRACE_DEFINE("[insert] found late-arriving cell in same bucket") +TURF_TRACE_DEFINE("[insert] set link on behalf of late-arriving cell") +TURF_TRACE_DEFINE("[insert] overflow") +TURF_TRACE_DEFINE("[beginTableMigrationToSize] called") +TURF_TRACE_DEFINE("[beginTableMigrationToSize] new migration already exists") +TURF_TRACE_DEFINE("[beginTableMigrationToSize] new migration already exists (double-checked)") +TURF_TRACE_DEFINE("[beginTableMigration] redirected while determining table size") +TURF_TRACE_DEFINE("[migrateRange] empty cell already redirected") +TURF_TRACE_DEFINE("[migrateRange] race to insert key") +TURF_TRACE_DEFINE("[migrateRange] race to insert value") +TURF_TRACE_DEFINE("[migrateRange] race inserted Redirect") +TURF_TRACE_DEFINE("[migrateRange] in-use cell already redirected") +TURF_TRACE_DEFINE("[migrateRange] racing update was erase") +TURF_TRACE_DEFINE("[migrateRange] race to update migrated value") +TURF_TRACE_DEFINE("[TableMigration::run] already ended") +TURF_TRACE_DEFINE("[TableMigration::run] detected end flag set") +TURF_TRACE_DEFINE("[TableMigration::run] destination overflow") +TURF_TRACE_DEFINE("[TableMigration::run] race to set m_overflowTableIndex") +TURF_TRACE_DEFINE("[TableMigration::run] out of migration units") +TURF_TRACE_DEFINE("[TableMigration::run] not the last worker") +TURF_TRACE_DEFINE("[TableMigration::run] a new TableMigration was already started") +TURF_TRACE_DEFINE("[TableMigration::run] overflow occured in a small map") +TURF_TRACE_DEFINE("[TableMigration::run] doubling subtree size after failure") +TURF_TRACE_DEFINE("[TableMigration::run] keeping same subtree size after failure") +TURF_TRACE_DEFINE("[FlatTreeMigration::run] already ended") +TURF_TRACE_DEFINE_END(Grampa, 37) + +} // namespace details +} // namespace junction diff --git a/junction/details/Grampa.h b/junction/details/Grampa.h new file mode 100644 index 0000000..b44a8c4 --- /dev/null +++ b/junction/details/Grampa.h @@ -0,0 +1,875 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_DETAILS_GRAMPA_H +#define JUNCTION_DETAILS_GRAMPA_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace junction { +namespace details { + +#if JUNCTION_TRACK_GRAMPA_STATS +struct GrampaCounter { + turf::Atomic total; + turf::Atomic current; + + void increment() { + total.fetchAdd(1, turf::Relaxed); + current.fetchAdd(1, turf::Relaxed); + } + + void decrement() { + current.fetchSub(1, turf::Relaxed); + } +}; + +struct GrampaStats { + GrampaCounter numTables; + GrampaCounter numTableMigrations; + GrampaCounter numFlatTrees; + GrampaCounter numFlatTreeMigrations; + + static GrampaStats Instance; // Zero-initialized +}; +#endif + +TURF_TRACE_DECLARE(Grampa, 37) + +template +struct Grampa { + typedef typename Map::Hash Hash; + typedef typename Map::Value Value; + typedef typename Map::KeyTraits KeyTraits; + typedef typename Map::ValueTraits ValueTraits; + + static const ureg RedirectFlatTree = 1; + static const ureg InitialSize = 8; + static const ureg TableMigrationUnitSize = 32; + static const ureg FlatTreeMigrationUnitSize = 32; + static const ureg LinearSearchLimit = 128; + static const ureg CellsInUseSample = LinearSearchLimit; + TURF_STATIC_ASSERT(LinearSearchLimit > 0 && LinearSearchLimit < 256); // Must fit in CellGroup::links + TURF_STATIC_ASSERT(CellsInUseSample > 0 && CellsInUseSample <= LinearSearchLimit); // Limit sample to failed search chain + + static const ureg MinTableSize = 8; + static const ureg LeafSizeBits = 10; + static const ureg LeafSize = (ureg(1) << LeafSizeBits); + + struct Cell { + // If value == Redirect, threads participate in the jobCoordinator. + turf::Atomic hash; + turf::Atomic value; + }; + + struct CellGroup { + // Every cell in the table actually represents a bucket of cells, all linked together in a probe chain. + // Each cell in the probe chain is located within the table itself. + // "deltas" determines the index of the next cell in the probe chain. + // The first cell in the chain is the one that was hashed. It may or may not actually belong in the bucket. + // The "second" cell in the chain is given by deltas 0 - 3. It's guaranteed to belong in the bucket. + // All subsequent cells in the chain is given by deltas 4 - 7. Also guaranteed to belong in the bucket. + turf::Atomic deltas[8]; + Cell cells[4]; + }; + + struct Table { + // unsafeRangeShift determines how many slots are occupied by this Table in the flattree. + // The range of hashes stored in this table is given by (1 << shift). + // eg. If the entire map is stored in a single table, then Table::shift == HASH_BITS. + // If the entire map is stored in two tables, then Table::shift == (HASH_BITS - 1) for each table. + // FlatTree::shift is always <= Table::shift for all the tables it contains. + const ureg sizeMask; // a power of two minus one + const Hash baseHash; + const ureg unsafeRangeShift; + junction::striped::ManualResetEvent isPublished; // To prevent publishing a subtree before its parent is published (happened in testing) + junction::striped::Mutex mutex; // to DCLI the TableMigration (stored in the jobCoordinator) + SimpleJobCoordinator jobCoordinator; // makes all blocked threads participate in the migration + + Table(ureg sizeMask, Hash baseHash, ureg unsafeRangeShift) : sizeMask(sizeMask), baseHash(baseHash), unsafeRangeShift(unsafeRangeShift) { + } + + static Table* create(ureg tableSize, ureg baseHash, ureg unsafeShift) { + TURF_ASSERT(turf::util::isPowerOf2(tableSize)); + TURF_ASSERT(unsafeShift > 0 && unsafeShift <= sizeof(Hash) * 8); + TURF_ASSERT(tableSize >= 4); + ureg numGroups = tableSize >> 2; + Table* table = (Table*) TURF_HEAP.alloc(sizeof(Table) + sizeof(CellGroup) * numGroups); + new(table) Table(tableSize - 1, baseHash, (u8) unsafeShift); + for (ureg i = 0; i < numGroups; i++) { + CellGroup* group = table->getCellGroups() + i; + for (ureg j = 0; j < 4; j++) { + group->deltas[j].storeNonatomic(0); + group->deltas[j + 4].storeNonatomic(0); + group->cells[j].hash.storeNonatomic(KeyTraits::NullHash); + group->cells[j].value.storeNonatomic(Value(ValueTraits::NullValue)); + } + } +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numTables.increment(); +#endif + return table; + } + + void destroy() { +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numTables.decrement(); +#endif + this->Table::~Table(); + TURF_HEAP.free(this); + } + + CellGroup* getCellGroups() const { + return (CellGroup*) (this + 1); + } + + ureg getNumMigrationUnits() const { + return sizeMask / TableMigrationUnitSize + 1; + } + }; + + class TableMigration : public SimpleJobCoordinator::Job { + public: + struct Source { + Table* table; + turf::Atomic sourceIndex; + }; + + Map& m_map; + Hash m_baseHash; // The lowest possible hash value in this subtree; determines index in flattree. + // If m_numDestinations == 1, m_shift == 0. + // Otherwise, m_shift tells (indirectly) the size of the flattree in which our subtree would exactly fit: 1 << (HASH_BITS - m_shift). + // This ensures that m_shift is always less than sizeof(Hash) * 8, so that shifting by m_shift is not undefined behavior. + // To determine the subtree index for a hash during migration, we use: (hash >> m_shift) & (m_numDestinations - 1) + // A mask is used since we are only migrating a subtree -- not necessarily the entire map. + ureg m_safeShift; + turf::Atomic m_workerStatus; // number of workers + end flag + turf::Atomic m_overflowTableIndex; + turf::Atomic m_unitsRemaining; + ureg m_numSources; + ureg m_numDestinations; // The size of the subtree being created. Some table pointers may be repeated. + + TableMigration(Map& map) : m_map(map) { + } + + static TableMigration* create(Map& map, ureg numSources, ureg numDestinations) { + TableMigration* migration = (TableMigration*) TURF_HEAP.alloc(sizeof(TableMigration) + sizeof(TableMigration::Source) * numSources + sizeof(Table*) * numDestinations); + new(migration) TableMigration(map); + migration->m_workerStatus.storeNonatomic(0); + migration->m_overflowTableIndex.storeNonatomic(-1); + migration->m_unitsRemaining.storeNonatomic(0); + migration->m_numSources = numSources; + migration->m_numDestinations = numDestinations; + // Caller is responsible for filling in source & destination pointers +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numTableMigrations.increment(); +#endif + return migration; + } + + virtual ~TableMigration() TURF_OVERRIDE { + } + + void destroy() { +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numTableMigrations.decrement(); +#endif + // Destroy all source tables. + for (ureg i = 0; i < m_numSources; i++) + if (getSources()[i].table) + getSources()[i].table->destroy(); + // Delete the migration object itself. + this->TableMigration::~TableMigration(); + TURF_HEAP.free(this); + } + + ureg getUnsafeShift() const { + return m_safeShift ? m_safeShift : (sizeof(Hash) * 8); + } + + Source* getSources() const { + return (Source*) (this + 1); + } + + Table** getDestinations() const { + return (Table**) (getSources() + m_numSources); + } + + sreg migrateRange(Table* srcTable, ureg startIdx); + virtual void run() TURF_OVERRIDE; + }; + + class FlatTreeMigration; + + struct FlatTree { + // The size of the flattree is 1 << 64 - HASH_BITS. + // Or, stated another way, (Hash(-1) >> shift) + 1. + // To determine the flattree index for a given hash, we simply use: (hash >> shift) + // Smaller shift == more significant bits used as an index == bigger flattree. + // For example, the simplest flattree has only two entries, and only the most significant + // bit of each hash is used as the flattree index. In that case, shift == HASH_BITS - 1. + // Each time the flattree doubles in size, shift decreases by 1. + const ureg safeShift; + junction::striped::Mutex mutex; + FlatTreeMigration* migration; // Protected by mutex + + FlatTree(ureg safeShift) : safeShift(safeShift), migration(NULL) { + // A FlatTree always has at least two tables, so the shift is always safe. + TURF_ASSERT(safeShift < sizeof(Hash) * 8); + } + + static FlatTree* create(ureg safeShift) { + // A flattree always has at least two tables, so the shift is always safe. + TURF_ASSERT(safeShift < sizeof(Hash) * 8); + ureg numLeaves = (Hash(-1) >> safeShift) + 1; + FlatTree* flatTree = (FlatTree*) TURF_HEAP.alloc(sizeof(FlatTree) + sizeof(turf::Atomic) * numLeaves); + new(flatTree) FlatTree(safeShift); + // Caller will initialize flatTree->getTables() +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numFlatTrees.increment(); +#endif + return flatTree; + } + + void destroy() { +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numFlatTrees.decrement(); +#endif + this->FlatTree::~FlatTree(); + TURF_HEAP.free(this); + } + + turf::Atomic* getTables() const { + return (turf::Atomic*) (this + 1); + } + + ureg getSize() const { + return (Hash(-1) >> safeShift) + 1; + } + + ureg getNumMigrationUnits() const { + ureg sizeMask = Hash(-1) >> safeShift; + return sizeMask / FlatTreeMigrationUnitSize + 1; + } + }; + + class FlatTreeMigration : public SimpleJobCoordinator::Job { + public: + Map& m_map; + FlatTree* m_source; + FlatTree* m_destination; + turf::Atomic m_workerStatus; + turf::Atomic m_sourceIndex; + turf::Atomic m_unitsRemaining; + junction::striped::ManualResetEvent m_completed; + + FlatTreeMigration(Map& map, FlatTree* flatTree, ureg shift) : m_map(map) { + m_source = flatTree; + m_destination = FlatTree::create(shift); + m_workerStatus.storeNonatomic(0); + m_sourceIndex.storeNonatomic(0); + m_unitsRemaining.storeNonatomic(flatTree->getNumMigrationUnits()); +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numFlatTreeMigrations.increment(); +#endif + } + + virtual ~FlatTreeMigration() TURF_OVERRIDE { +#if JUNCTION_TRACK_GRAMPA_STATS + GrampaStats::Instance.numFlatTreeMigrations.decrement(); +#endif + // Delete source flattree. + m_source->destroy(); + } + + void destroy() { + delete this; + } + + virtual void run() TURF_OVERRIDE; + }; + + static void garbageCollectTable(Table* table) { + TURF_ASSERT(table); + DefaultQSBR.enqueue(&Table::destroy, table); + } + + static void garbageCollectFlatTree(FlatTree* flatTree) { + TURF_ASSERT(flatTree); + DefaultQSBR.enqueue(&FlatTree::destroy, flatTree); + } + + static Cell* find(Hash hash, Table* table, ureg sizeMask) { + TURF_TRACE(Grampa, 0, "[find] called", uptr(table), hash); + TURF_ASSERT(table); + TURF_ASSERT(hash != KeyTraits::NullHash); + // Optimistically check hashed cell even though it might belong to another bucket + ureg idx = hash & sizeMask; + CellGroup* group = table->getCellGroups() + (idx >> 2); + Cell* cell = group->cells + (idx & 3); + Hash probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == hash) { + TURF_TRACE(Grampa, 1, "[find] found existing cell optimistically", uptr(table), idx); + return cell; + } else if (probeHash == KeyTraits::NullHash) { + return cell = NULL; + } + // Follow probe chain for our bucket + u8 delta = group->deltas[idx & 3].load(turf::Relaxed); + while (delta) { + idx = (idx + delta) & sizeMask; + group = table->getCellGroups() + (idx >> 2); + cell = group->cells + (idx & 3); + Hash probeHash = cell->hash.load(turf::Relaxed); + // Note: probeHash might actually be NULL due to memory reordering of a concurrent insert, + // but we don't check for it. We just follow the probe chain. + if (probeHash == hash) { + TURF_TRACE(Grampa, 2, "[find] found existing cell", uptr(table), idx); + return cell; + } + delta = group->deltas[(idx & 3) + 4].load(turf::Relaxed); + } + // End of probe chain, not found + return NULL; + } + + // FIXME: Possible optimization: Dedicated insert for migration? It wouldn't check for InsertResult_AlreadyFound. + enum InsertResult { + InsertResult_AlreadyFound, + InsertResult_InsertedNew, + InsertResult_Overflow + }; + static InsertResult insert(Hash hash, Table* table, ureg sizeMask, Cell*& cell, ureg& overflowIdx) { + TURF_TRACE(Grampa, 3, "[insert] called", uptr(table), hash); + TURF_ASSERT(table); + TURF_ASSERT(hash != KeyTraits::NullHash); + ureg idx = hash; + + // Check hashed cell first, though it may not even belong to the bucket. + CellGroup* group = table->getCellGroups() + ((idx & sizeMask) >> 2); + cell = group->cells + (idx & 3); + Hash probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == KeyTraits::NullHash) { + if (cell->hash.compareExchangeStrong(probeHash, hash, turf::Relaxed)) { + TURF_TRACE(Grampa, 4, "[insert] reserved first cell", uptr(table), idx); + // There are no links to set. We're done. + return InsertResult_InsertedNew; + } else { + TURF_TRACE(Grampa, 5, "[insert] race to reserve first cell", uptr(table), idx); + // Fall through to check if it was the same hash... + } + } + if (probeHash == hash) { + TURF_TRACE(Grampa, 6, "[insert] found in first cell", uptr(table), idx); + return InsertResult_AlreadyFound; + } + + // Follow the link chain for this bucket. + ureg maxIdx = idx + sizeMask; + ureg linkLevel = 0; + turf::Atomic* prevLink; + for (;;) { + followLink: + prevLink = group->deltas + ((idx & 3) + linkLevel); + linkLevel = 4; + u8 probeDelta = prevLink->load(turf::Relaxed); + if (probeDelta) { + idx += probeDelta; + // Check the hash for this cell. + group = table->getCellGroups() + ((idx & sizeMask) >> 2); + cell = group->cells + (idx & 3); + probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == KeyTraits::NullHash) { + // Cell was linked, but hash is not visible yet. + // We could avoid this case (and guarantee it's visible) using acquire & release, but instead, + // just poll until it becomes visible. + TURF_TRACE(Grampa, 7, "[insert] race to read hash", uptr(table), idx); + do { + probeHash = cell->hash.load(turf::Acquire); + } while (probeHash == KeyTraits::NullHash); + } + TURF_ASSERT(((probeHash ^ hash) & sizeMask) == 0); // Only hashes in same bucket can be linked + if (probeHash == hash) { + TURF_TRACE(Grampa, 8, "[insert] found in probe chain", uptr(table), idx); + return InsertResult_AlreadyFound; + } + } else { + // Reached the end of the link chain for this bucket. + // Switch to linear probing until we reserve a new cell or find a late-arriving cell in the same bucket. + ureg prevLinkIdx = idx; + TURF_ASSERT(sreg(maxIdx - idx) >= 0); // Nobody would have linked an idx that's out of range. + ureg linearProbesRemaining = turf::util::min(maxIdx - idx, LinearSearchLimit); + while (linearProbesRemaining-- > 0) { + idx++; + group = table->getCellGroups() + ((idx & sizeMask) >> 2); + cell = group->cells + (idx & 3); + probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == KeyTraits::NullHash) { + // It's an empty cell. Try to reserve it. + if (cell->hash.compareExchangeStrong(probeHash, hash, turf::Relaxed)) { + // Success. We've reserved the cell. Link it to previous cell in same bucket. + TURF_TRACE(Grampa, 9, "[insert] reserved cell", uptr(table), idx); + TURF_ASSERT(probeDelta == 0); + u8 desiredDelta = idx - prevLinkIdx; + // Note: another thread could actually set the link on our behalf (see below). +#if TURF_WITH_ASSERTS + probeDelta = prevLink->exchange(desiredDelta, turf::Relaxed); + TURF_ASSERT(probeDelta == 0 || probeDelta == desiredDelta); +#else + prevLink->store(desiredDelta, turf::Relaxed); +#endif + return InsertResult_InsertedNew; + } else { + TURF_TRACE(Grampa, 10, "[insert] race to reserve cell", uptr(table), idx); + // Fall through to check if it's the same hash... + } + } + Hash x = (probeHash ^ hash); + // Check for same hash. + if (!x) { + TURF_TRACE(Grampa, 11, "[insert] found outside probe chain", uptr(table), idx); + return InsertResult_AlreadyFound; + } + // Check for same bucket. + if ((x & sizeMask) == 0) { + TURF_TRACE(Grampa, 12, "[insert] found late-arriving cell in same bucket", uptr(table), idx); + // Attempt to set the link on behalf of the late-arriving cell. + // This is usually redundant, but if we don't attempt to set the late-arriving cell's link here, + // there's no guarantee that our own link chain will be well-formed by the time this function returns. + // (Indeed, subsequent lookups sometimes failed during testing, for this exact reason.) + u8 desiredDelta = idx - prevLinkIdx; +#if TURF_WITH_ASSERTS + probeDelta = prevLink->exchange(desiredDelta, turf::Relaxed); + TURF_ASSERT(probeDelta == 0 || probeDelta == desiredDelta); + if (probeDelta == 0) + TURF_TRACE(Grampa, 13, "[insert] set link on behalf of late-arriving cell", uptr(table), idx); +#else + prevLink->store(desiredDelta, turf::Relaxed); +#endif + goto followLink; // Try to follow link chain for the bucket again. + } + // Continue linear search... + } + // Table is too full to insert. + overflowIdx = idx + 1; + TURF_TRACE(Grampa, 14, "[insert] overflow", uptr(table), overflowIdx); + return InsertResult_Overflow; + } + } + } + + static void beginTableMigrationToSize(Map& map, Table* table, ureg nextTableSize, ureg splitShift) { + // Create new migration by DCLI. + TURF_TRACE(Grampa, 15, "[beginTableMigrationToSize] called", 0, 0); + SimpleJobCoordinator::Job* job = table->jobCoordinator.loadConsume(); + if (job) { + TURF_TRACE(Grampa, 16, "[beginTableMigrationToSize] new migration already exists", 0, 0); + } else { + turf::LockGuard guard(table->mutex); + job = table->jobCoordinator.loadConsume(); // Non-atomic would be sufficient, but that's OK. + if (job) { + TURF_TRACE(Grampa, 17, "[beginTableMigrationToSize] new migration already exists (double-checked)", 0, 0); + } else { + // Create new migration. + ureg numDestinations = ureg(1) << splitShift; + TableMigration* migration = TableMigration::create(map, 1, numDestinations); + migration->m_baseHash = table->baseHash; + ureg migrationShift = table->unsafeRangeShift - splitShift; + migration->m_safeShift = (migrationShift < sizeof(Hash) * 8) ? migrationShift : 0; + migration->m_unitsRemaining.storeNonatomic(table->getNumMigrationUnits()); + migration->getSources()[0].table = table; + migration->getSources()[0].sourceIndex.storeNonatomic(0); + ureg subRangeShift = table->unsafeRangeShift - splitShift; // subRangeShift is also "unsafe" (possibly represents entire range) + ureg hashOffsetDelta = subRangeShift < (sizeof(Hash) * 8) ? (ureg(1) << subRangeShift) : 0; + for (ureg i = 0; i < numDestinations; i++) { + migration->getDestinations()[i] = Table::create(nextTableSize, table->baseHash + hashOffsetDelta * i, subRangeShift); + } + // Publish the new migration. + table->jobCoordinator.storeRelease(migration); + } + } + } + + static void beginTableMigration(Map& map, Table* table, ureg overflowIdx) { + // Estimate number of cells in use based on a small sample. + ureg sizeMask = table->sizeMask; + ureg idx = overflowIdx - CellsInUseSample; + ureg inUseCells = 0; + for (ureg linearProbesRemaining = CellsInUseSample; linearProbesRemaining > 0; linearProbesRemaining--) { + CellGroup* group = table->getCellGroups() + ((idx & sizeMask) >> 2); + Cell* cell = group->cells + (idx & 3); + Value value = cell->value.load(turf::Relaxed); + if (value == Value(ValueTraits::Redirect)) { + // Another thread kicked off the jobCoordinator. The caller will participate upon return. + TURF_TRACE(Grampa, 18, "[beginTableMigration] redirected while determining table size", 0, 0); + return; + } + if (value != Value(ValueTraits::NullValue)) + inUseCells++; + idx++; + } + float inUseRatio = float(inUseCells) / CellsInUseSample; + float estimatedInUse = (sizeMask + 1) * inUseRatio; + ureg nextTableSize = turf::util::roundUpPowerOf2(ureg(estimatedInUse * 2)); + // FIXME: Support migrating to smaller tables. + nextTableSize = turf::util::max(nextTableSize, sizeMask + 1); + // Split into multiple tables if necessary. + ureg splitShift = 0; + while (nextTableSize > LeafSize) { + splitShift++; + nextTableSize >>= 1; + } + beginTableMigrationToSize(map, table, nextTableSize, splitShift); + } + + static FlatTreeMigration* createFlatTreeMigration(Map& map, FlatTree* flatTree, ureg shift) { + turf::LockGuard guard(flatTree->mutex); + if (!flatTree->migration) { + flatTree->migration = new FlatTreeMigration(map, flatTree, shift); + } + return flatTree->migration; + } + + static FlatTreeMigration* getExistingFlatTreeMigration(FlatTree* flatTree) { + turf::LockGuard guard(flatTree->mutex); + TURF_ASSERT(flatTree->migration); // Must already exist! + return flatTree->migration; + } +}; // Grampa + +// Return index of the destination table that overflowed, or -1 if none +template +sreg Grampa::TableMigration::migrateRange(Table* srcTable, ureg startIdx) { + ureg srcSizeMask = srcTable->sizeMask; + ureg safeShift = m_safeShift; + Table** dstLeafs = getDestinations(); + ureg dstLeafMask = m_numDestinations - 1; + ureg endIdx = turf::util::min(startIdx + TableMigrationUnitSize, srcSizeMask + 1); + // Iterate over source range. + for (ureg srcIdx = startIdx; srcIdx < endIdx; srcIdx++) { + CellGroup* srcGroup = srcTable->getCellGroups() + ((srcIdx & srcSizeMask) >> 2); + Cell* srcCell = srcGroup->cells + (srcIdx & 3); + Hash srcHash; + Value srcValue; + // Fetch the srcHash and srcValue. + for (;;) { + srcHash = srcCell->hash.load(turf::Relaxed); + if (srcHash == KeyTraits::NullHash) { + // An unused cell. Try to put a Redirect marker in its value. + srcValue = srcCell->value.compareExchange(Value(ValueTraits::NullValue), Value(ValueTraits::Redirect), turf::Relaxed); + if (srcValue == Value(ValueTraits::Redirect)) { + // srcValue is already marked Redirect due to previous incomplete migration. + TURF_TRACE(Grampa, 19, "[migrateRange] empty cell already redirected", uptr(srcTable), srcIdx); + break; + } + if (srcValue == Value(ValueTraits::NullValue)) + break; // Redirect has been placed. Break inner loop, continue outer loop. + TURF_TRACE(Grampa, 20, "[migrateRange] race to insert key", uptr(srcTable), srcIdx); + // Otherwise, somebody just claimed the cell. Read srcHash again... + } else { + // Check for deleted/uninitialized value. + srcValue = srcCell->value.load(turf::Relaxed); + if (srcValue == Value(ValueTraits::NullValue)) { + // Try to put a Redirect marker. + if (srcCell->value.compareExchangeStrong(srcValue, Value(ValueTraits::Redirect), turf::Relaxed)) + break; // Redirect has been placed. Break inner loop, continue outer loop. + TURF_TRACE(Grampa, 21, "[migrateRange] race to insert value", uptr(srcTable), srcIdx); + if (srcValue == Value(ValueTraits::Redirect)) { + // FIXME: I don't think this will happen. Investigate & change to assert + TURF_TRACE(Grampa, 22, "[migrateRange] race inserted Redirect", uptr(srcTable), srcIdx); + break; + } + } else if (srcValue == Value(ValueTraits::Redirect)) { + // srcValue is already marked Redirect due to previous incomplete migration. + TURF_TRACE(Grampa, 23, "[migrateRange] in-use cell already redirected", uptr(srcTable), srcIdx); + break; + } + + // We've got a key/value pair to migrate. + // Reserve a destination cell in dstTable. + TURF_ASSERT(srcHash != KeyTraits::NullHash); + TURF_ASSERT(srcValue != Value(ValueTraits::NullValue)); + TURF_ASSERT(srcValue != Value(ValueTraits::Redirect)); + ureg destLeafIndex = (srcHash >> safeShift) & dstLeafMask; + Table* dstLeaf = dstLeafs[destLeafIndex]; + Cell* dstCell; + ureg overflowIdx; + InsertResult result = insert(srcHash, dstLeaf, dstLeaf->sizeMask, dstCell, overflowIdx); + // During migration, a hash can only exist in one place among all the source tables, + // and it is only migrated by one thread. Therefore, the hash will never already exist + // in the destination table: + TURF_ASSERT(result != InsertResult_AlreadyFound); + if (result == InsertResult_Overflow) { + // Destination overflow. + // This can happen for several reasons. For example, the source table could have + // existed of all deleted cells when it overflowed, resulting in a small destination + // table size, but then another thread could re-insert all the same hashes + // before the migration completed. + // Caller will cancel the current migration and begin a new one. + return destLeafIndex; + } + // Migrate the old value to the new cell. + for (;;) { + // Copy srcValue to the destination. + dstCell->value.store(srcValue, turf::Relaxed); + // Try to place a Redirect marker in srcValue. + Value doubleCheckedSrcValue = srcCell->value.compareExchange(srcValue, Value(ValueTraits::Redirect), turf::Relaxed); + TURF_ASSERT(doubleCheckedSrcValue != Value(ValueTraits::Redirect)); // Only one thread can redirect a cell at a time. + if (doubleCheckedSrcValue == srcValue) { + // No racing writes to the src. We've successfully placed the Redirect marker. + // srcValue was non-NULL when we decided to migrate it, but it may have changed to NULL + // by a late-arriving erase. + if (srcValue == Value(ValueTraits::NullValue)) + TURF_TRACE(Grampa, 24, "[migrateRange] racing update was erase", uptr(srcTable), srcIdx); + break; + } + // There was a late-arriving write (or erase) to the src. Migrate the new value and try again. + TURF_TRACE(Grampa, 25, "[migrateRange] race to update migrated value", uptr(srcTable), srcIdx); + srcValue = doubleCheckedSrcValue; + } + // Cell successfully migrated. Proceed to next source cell. + break; + } + } + } + // Range has been migrated successfully. + return -1; +} + +template +void Grampa::TableMigration::run() { + // Conditionally increment the shared # of workers. + ureg probeStatus = m_workerStatus.load(turf::Relaxed); + do { + if (probeStatus & 1) { + // End flag is already set, so do nothing. + TURF_TRACE(Grampa, 26, "[TableMigration::run] already ended", uptr(this), 0); + return; + } + } while (!m_workerStatus.compareExchangeWeak(probeStatus, probeStatus + 2, turf::Relaxed, turf::Relaxed)); + // # of workers has been incremented, and the end flag is clear. + TURF_ASSERT((probeStatus & 1) == 0); + + // Iterate over all source tables. + Source* sources = getSources(); + for (ureg s = 0; s < m_numSources; s++) { + Source& source = sources[s]; + // Loop over all migration units in this source table. + for (;;) { + if (m_workerStatus.load(turf::Relaxed) & 1) { + TURF_TRACE(Grampa, 27, "[TableMigration::run] detected end flag set", uptr(this), 0); + goto endMigration; + } + ureg startIdx = source.sourceIndex.fetchAdd(TableMigrationUnitSize, turf::Relaxed); + if (startIdx >= source.table->sizeMask + 1) + break; // No more migration units in this table. Try next source table. + sreg overflowTableIndex = migrateRange(source.table, startIdx); + if (overflowTableIndex >= 0) { + // *** FAILED MIGRATION *** + // TableMigration failed due to destination table overflow. + // No other thread can declare the migration successful at this point, because *this* unit will never complete, hence m_unitsRemaining won't reach zero. + // However, multiple threads can independently detect a failed migration at the same time. + TURF_TRACE(Grampa, 28, "[TableMigration::run] destination overflow", uptr(source.table), uptr(startIdx)); + // The reason we store overflowTableIndex in a shared variable is because we must flush all the worker threads before + // we can safely deal with the overflow. Therefore, the thread that detects the failure is often different from the thread + // that deals with it. + // Store overflowTableIndex unconditionally; racing writes should be rare, and it doesn't matter which one wins. + sreg oldIndex = m_overflowTableIndex.exchange(overflowTableIndex, turf::Relaxed); + if (oldIndex >= 0) + TURF_TRACE(Grampa, 29, "[TableMigration::run] race to set m_overflowTableIndex", uptr(overflowTableIndex), uptr(oldIndex)); + m_workerStatus.fetchOr(1, turf::Relaxed); + goto endMigration; + } + sreg prevRemaining = m_unitsRemaining.fetchSub(1, turf::Relaxed); + TURF_ASSERT(prevRemaining > 0); + if (prevRemaining == 1) { + // *** SUCCESSFUL MIGRATION *** + // That was the last chunk to migrate. + m_workerStatus.fetchOr(1, turf::Relaxed); + goto endMigration; + } + } + } + TURF_TRACE(Grampa, 30, "[TableMigration::run] out of migration units", uptr(this), 0); + +endMigration: + // Decrement the shared # of workers. + probeStatus = m_workerStatus.fetchSub(2, turf::AcquireRelease); // Ensure all modifications are visible to the thread that will publish + if (probeStatus >= 4) { + // There are other workers remaining. Return here so that only the very last worker will proceed. + TURF_TRACE(Grampa, 31, "[TableMigration::run] not the last worker", uptr(this), uptr(probeStatus)); + return; + } + + // We're the very last worker thread. + // Perform the appropriate post-migration step depending on whether the migration succeeded or failed. + TURF_ASSERT(probeStatus == 3); + sreg overflowTableIndex = m_overflowTableIndex.loadNonatomic(); // No racing writes at this point + if (overflowTableIndex < 0) { + // The migration succeeded. This is the most likely outcome. Publish the new subtree. + m_map.publishTableMigration(this); + // End the jobCoodinator. + sources[0].table->jobCoordinator.end(); + } else { + // The migration failed due to the overflow of a destination table. + Table* origTable = sources[0].table; + ureg count = ureg(1) << (origTable->unsafeRangeShift - getUnsafeShift()); + ureg lo = overflowTableIndex & ~(count - 1); + TURF_ASSERT(lo + count <= m_numDestinations); + turf::LockGuard guard(origTable->mutex); + SimpleJobCoordinator::Job* checkedJob = origTable->jobCoordinator.loadConsume(); + if (checkedJob != this) { + TURF_TRACE(Grampa, 32, "[TableMigration::run] a new TableMigration was already started", uptr(origTable), uptr(checkedJob)); + } else { + TableMigration* migration; + Table* overflowedTable = getDestinations()[overflowTableIndex]; + if (overflowedTable->sizeMask + 1 < LeafSize) { + // The entire map is contained in a small table. + TURF_TRACE(Grampa, 33, "[TableMigration::run] overflow occured in a small map", uptr(origTable), uptr(checkedJob)); + TURF_ASSERT(overflowedTable->unsafeRangeShift == sizeof(Hash) * 8); + TURF_ASSERT(overflowedTable->baseHash == 0); + TURF_ASSERT(m_numDestinations == 1); + TURF_ASSERT(m_baseHash == 0); + migration = TableMigration::create(m_map, m_numSources + 1, 1); + migration->m_baseHash = 0; + migration->m_safeShift = 0; + // Double the destination table size. + migration->getDestinations()[0] = Table::create((overflowedTable->sizeMask + 1) * 2, overflowedTable->baseHash, overflowedTable->unsafeRangeShift); + } else { + // The overflowed table is already the size of a leaf. Split it into two ranges. + if (count == 1) { + TURF_TRACE(Grampa, 34, "[TableMigration::run] doubling subtree size after failure", uptr(origTable), uptr(checkedJob)); + migration = TableMigration::create(m_map, m_numSources + 1, m_numDestinations * 2); + migration->m_baseHash = m_baseHash; + migration->m_safeShift = getUnsafeShift() - 1; + for (ureg i = 0; i < m_numDestinations; i++) { + migration->getDestinations()[i * 2] = getDestinations()[i]; + migration->getDestinations()[i * 2 + 1] = getDestinations()[i]; + } + count = 2; + } else { + TURF_TRACE(Grampa, 35, "[TableMigration::run] keeping same subtree size after failure", uptr(origTable), uptr(checkedJob)); + migration = TableMigration::create(m_map, m_numSources + 1, m_numDestinations); + migration->m_baseHash = m_baseHash; + migration->m_safeShift = m_safeShift; + memcpy(migration->getDestinations(), getDestinations(), m_numDestinations * sizeof(Table*)); + } + Table* splitTable1 = Table::create(LeafSize, origTable->baseHash, origTable->unsafeRangeShift - 1); + ureg i = 0; + for (; i < count / 2; i++) { + migration->getDestinations()[lo + i] = splitTable1; + } + ureg halfNumHashes = ureg(1) << (origTable->unsafeRangeShift - 1); + Table* splitTable2 = Table::create(LeafSize, origTable->baseHash + halfNumHashes, origTable->unsafeRangeShift - 1); + for (; i < count; i++) { + migration->getDestinations()[lo + i] = splitTable2; + } + } + // Transfer source tables to the new migration. + for (ureg i = 0; i < m_numSources; i++) { + migration->getSources()[i].table = getSources()[i].table; + migration->getSources()[i].sourceIndex.storeNonatomic(0); + getSources()[i].table = NULL; + } + migration->getSources()[m_numSources].table = overflowedTable; + migration->getSources()[m_numSources].sourceIndex.storeNonatomic(0); + // Calculate total number of migration units to move. + ureg unitsRemaining = 0; + for (ureg s = 0; s < migration->m_numSources; s++) + unitsRemaining += migration->getSources()[s].table->getNumMigrationUnits(); + migration->m_unitsRemaining.storeNonatomic(unitsRemaining); + // Publish the new migration. + origTable->jobCoordinator.storeRelease(migration); + } + } + + // We're done with this TableMigration. Queue it for GC. + DefaultQSBR.enqueue(&TableMigration::destroy, this); +} + +template +void Grampa::FlatTreeMigration::run() { + // Conditionally increment the shared # of workers. + ureg probeStatus = m_workerStatus.load(turf::Relaxed); + do { + if (probeStatus & 1) { + // End flag is already set, so do nothing. + TURF_TRACE(Grampa, 36, "[FlatTreeMigration::run] already ended", uptr(this), 0); + return; + } + } while (!m_workerStatus.compareExchangeWeak(probeStatus, probeStatus + 2, turf::Relaxed, turf::Relaxed)); + // # of workers has been incremented, and the end flag is clear. + TURF_ASSERT((probeStatus & 1) == 0); + + // Loop over all migration units + ureg srcSize = (Hash(-1) >> m_source->safeShift) + 1; + // FIXME: Support migration to smaller flattrees + TURF_ASSERT(m_destination->safeShift < m_source->safeShift); + ureg repeat = ureg(1) << (m_source->safeShift - m_destination->safeShift); + for (;;) { + ureg srcStart = m_sourceIndex.fetchAdd(FlatTreeMigrationUnitSize, turf::Relaxed); + if (srcStart >= srcSize) + break; // No more migration units in this flattree. + // Migrate this range + ureg srcEnd = turf::util::min(srcSize, srcStart + FlatTreeMigrationUnitSize); + ureg dst = srcStart * repeat; + for (ureg src = srcStart; src < srcEnd; src++) { + // Pointers in the source table can be changed at any time due to concurrent subtree publishing, + // so we need to exchange them with Redirect markers. + Table* t = m_source->getTables()[src].exchange((Table*) RedirectFlatTree, turf::Relaxed); + TURF_ASSERT(uptr(t) != RedirectFlatTree); + for (ureg r = repeat; r > 0; r--) { + m_destination->getTables()[dst].storeNonatomic(t); + dst++; + } + } + // Decrement m_unitsRemaining + sreg prevRemaining = m_unitsRemaining.fetchSub(1, turf::Relaxed); + if (prevRemaining == 1) { + // *** SUCCESSFUL MIGRATION *** + // That was the last chunk to migrate. + m_workerStatus.fetchOr(1, turf::Relaxed); + break; + } + } + + // Decrement the shared # of workers. + probeStatus = m_workerStatus.fetchSub(2, turf::AcquireRelease); // AcquireRelease makes all previous writes visible to the last worker thread. + if (probeStatus >= 4) { + // There are other workers remaining. Return here so that only the very last worker will proceed. + return; + } + + // We're the very last worker thread. + // Publish the new flattree. + TURF_ASSERT(probeStatus == 3); // End flag must be set + m_map.publishFlatTreeMigration(this); + m_completed.signal(); + + // We're done with this FlatTreeMigration. Queue it for GC. + DefaultQSBR.enqueue(&FlatTreeMigration::destroy, this); +} + +} // namespace details +} // namespace junction + +#endif // JUNCTION_DETAILS_GRAMPA_H diff --git a/junction/details/LeapFrog.cpp b/junction/details/LeapFrog.cpp new file mode 100644 index 0000000..3a7011e --- /dev/null +++ b/junction/details/LeapFrog.cpp @@ -0,0 +1,57 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include + +namespace junction { +namespace details { + +TURF_TRACE_DEFINE_BEGIN(LeapFrog, 33) // autogenerated by TidySource.py +TURF_TRACE_DEFINE("[find] called") +TURF_TRACE_DEFINE("[find] found existing cell optimistically") +TURF_TRACE_DEFINE("[find] found existing cell") +TURF_TRACE_DEFINE("[insert] called") +TURF_TRACE_DEFINE("[insert] reserved first cell") +TURF_TRACE_DEFINE("[insert] race to reserve first cell") +TURF_TRACE_DEFINE("[insert] found in first cell") +TURF_TRACE_DEFINE("[insert] race to read hash") +TURF_TRACE_DEFINE("[insert] found in probe chain") +TURF_TRACE_DEFINE("[insert] reserved cell") +TURF_TRACE_DEFINE("[insert] race to reserve cell") +TURF_TRACE_DEFINE("[insert] found outside probe chain") +TURF_TRACE_DEFINE("[insert] found late-arriving cell in same bucket") +TURF_TRACE_DEFINE("[insert] set link on behalf of late-arriving cell") +TURF_TRACE_DEFINE("[insert] overflow") +TURF_TRACE_DEFINE("[beginTableMigrationToSize] called") +TURF_TRACE_DEFINE("[beginTableMigrationToSize] new migration already exists") +TURF_TRACE_DEFINE("[beginTableMigrationToSize] new migration already exists (double-checked)") +TURF_TRACE_DEFINE("[beginTableMigration] redirected while determining table size") +TURF_TRACE_DEFINE("[migrateRange] empty cell already redirected") +TURF_TRACE_DEFINE("[migrateRange] race to insert key") +TURF_TRACE_DEFINE("[migrateRange] race to insert value") +TURF_TRACE_DEFINE("[migrateRange] race inserted Redirect") +TURF_TRACE_DEFINE("[migrateRange] in-use cell already redirected") +TURF_TRACE_DEFINE("[migrateRange] racing update was erase") +TURF_TRACE_DEFINE("[migrateRange] race to update migrated value") +TURF_TRACE_DEFINE("[TableMigration::run] already ended") +TURF_TRACE_DEFINE("[TableMigration::run] detected end flag set") +TURF_TRACE_DEFINE("[TableMigration::run] destination overflow") +TURF_TRACE_DEFINE("[TableMigration::run] race to set m_overflowed") +TURF_TRACE_DEFINE("[TableMigration::run] out of migration units") +TURF_TRACE_DEFINE("[TableMigration::run] not the last worker") +TURF_TRACE_DEFINE("[TableMigration::run] a new TableMigration was already started") +TURF_TRACE_DEFINE_END(LeapFrog, 33) + +} // namespace details +} // namespace junction diff --git a/junction/details/LeapFrog.h b/junction/details/LeapFrog.h new file mode 100644 index 0000000..7b1e395 --- /dev/null +++ b/junction/details/LeapFrog.h @@ -0,0 +1,561 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_DETAILS_LEAPFROG_H +#define JUNCTION_DETAILS_LEAPFROG_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace junction { +namespace details { + +TURF_TRACE_DECLARE(LeapFrog, 33) + +template +struct LeapFrog { + typedef typename Map::Hash Hash; + typedef typename Map::Value Value; + typedef typename Map::KeyTraits KeyTraits; + typedef typename Map::ValueTraits ValueTraits; + + static const ureg InitialSize = 8; + static const ureg TableMigrationUnitSize = 32; + static const ureg LinearSearchLimit = 128; + static const ureg CellsInUseSample = LinearSearchLimit; + TURF_STATIC_ASSERT(LinearSearchLimit > 0 && LinearSearchLimit < 256); // Must fit in CellGroup::links + TURF_STATIC_ASSERT(CellsInUseSample > 0 && CellsInUseSample <= LinearSearchLimit); // Limit sample to failed search chain + + struct Cell { + turf::Atomic hash; + turf::Atomic value; + }; + + struct CellGroup { + // Every cell in the table actually represents a bucket of cells, all linked together in a probe chain. + // Each cell in the probe chain is located within the table itself. + // "deltas" determines the index of the next cell in the probe chain. + // The first cell in the chain is the one that was hashed. It may or may not actually belong in the bucket. + // The "second" cell in the chain is given by deltas 0 - 3. It's guaranteed to belong in the bucket. + // All subsequent cells in the chain is given by deltas 4 - 7. Also guaranteed to belong in the bucket. + turf::Atomic deltas[8]; + Cell cells[4]; + }; + + struct Table { + const ureg sizeMask; // a power of two minus one + turf::Mutex mutex; // to DCLI the TableMigration (stored in the jobCoordinator) + SimpleJobCoordinator jobCoordinator; // makes all blocked threads participate in the migration + + Table(ureg sizeMask) : sizeMask(sizeMask) { + } + + static Table* create(ureg tableSize) { + TURF_ASSERT(turf::util::isPowerOf2(tableSize)); + TURF_ASSERT(tableSize >= 4); + ureg numGroups = tableSize >> 2; + Table* table = (Table*) TURF_HEAP.alloc(sizeof(Table) + sizeof(CellGroup) * numGroups); + new(table) Table(tableSize - 1); + for (ureg i = 0; i < numGroups; i++) { + CellGroup* group = table->getCellGroups() + i; + for (ureg j = 0; j < 4; j++) { + group->deltas[j].storeNonatomic(0); + group->deltas[j + 4].storeNonatomic(0); + group->cells[j].hash.storeNonatomic(KeyTraits::NullHash); + group->cells[j].value.storeNonatomic(Value(ValueTraits::NullValue)); + } + } + return table; + } + + void destroy() { + this->Table::~Table(); + TURF_HEAP.free(this); + } + + CellGroup* getCellGroups() const { + return (CellGroup*) (this + 1); + } + + ureg getNumMigrationUnits() const { + return sizeMask / TableMigrationUnitSize + 1; + } + }; + + class TableMigration : public SimpleJobCoordinator::Job { + public: + struct Source { + Table* table; + turf::Atomic sourceIndex; + }; + + Map& m_map; + Table* m_destination; + turf::Atomic m_workerStatus; // number of workers + end flag + turf::Atomic m_overflowed; + turf::Atomic m_unitsRemaining; + ureg m_numSources; + + TableMigration(Map& map) : m_map(map) { + } + + static TableMigration* create(Map& map, ureg numSources) { + TableMigration* migration = (TableMigration*) TURF_HEAP.alloc(sizeof(TableMigration) + sizeof(TableMigration::Source) * numSources); + new(migration) TableMigration(map); + migration->m_workerStatus.storeNonatomic(0); + migration->m_overflowed.storeNonatomic(false); + migration->m_unitsRemaining.storeNonatomic(0); + migration->m_numSources = numSources; + // Caller is responsible for filling in sources & destination + return migration; + } + + virtual ~TableMigration() TURF_OVERRIDE { + } + + void destroy() { + // Destroy all source tables. + for (ureg i = 0; i < m_numSources; i++) + if (getSources()[i].table) + getSources()[i].table->destroy(); + // Delete the migration object itself. + this->TableMigration::~TableMigration(); + TURF_HEAP.free(this); + } + + Source* getSources() const { + return (Source*) (this + 1); + } + + bool migrateRange(Table* srcTable, ureg startIdx); + virtual void run() TURF_OVERRIDE; + }; + + static Cell* find(Hash hash, Table* table) { + TURF_TRACE(LeapFrog, 0, "[find] called", uptr(table), hash); + TURF_ASSERT(table); + TURF_ASSERT(hash != KeyTraits::NullHash); + ureg sizeMask = table->sizeMask; + // Optimistically check hashed cell even though it might belong to another bucket + ureg idx = hash & sizeMask; + CellGroup* group = table->getCellGroups() + (idx >> 2); + Cell* cell = group->cells + (idx & 3); + Hash probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == hash) { + TURF_TRACE(LeapFrog, 1, "[find] found existing cell optimistically", uptr(table), idx); + return cell; + } else if (probeHash == KeyTraits::NullHash) { + return cell = NULL; + } + // Follow probe chain for our bucket + u8 delta = group->deltas[idx & 3].load(turf::Relaxed); + while (delta) { + idx = (idx + delta) & sizeMask; + group = table->getCellGroups() + (idx >> 2); + cell = group->cells + (idx & 3); + Hash probeHash = cell->hash.load(turf::Relaxed); + // Note: probeHash might actually be NULL due to memory reordering of a concurrent insert, + // but we don't check for it. We just follow the probe chain. + if (probeHash == hash) { + TURF_TRACE(LeapFrog, 2, "[find] found existing cell", uptr(table), idx); + return cell; + } + delta = group->deltas[(idx & 3) + 4].load(turf::Relaxed); + } + // End of probe chain, not found + return NULL; + } + + // FIXME: Possible optimization: Dedicated insert for migration? It wouldn't check for InsertResult_AlreadyFound. + enum InsertResult { + InsertResult_AlreadyFound, + InsertResult_InsertedNew, + InsertResult_Overflow + }; + static InsertResult insert(Hash hash, Table* table, Cell*& cell, ureg& overflowIdx) { + TURF_TRACE(LeapFrog, 3, "[insert] called", uptr(table), hash); + TURF_ASSERT(table); + TURF_ASSERT(hash != KeyTraits::NullHash); + ureg sizeMask = table->sizeMask; + ureg idx = hash; + + // Check hashed cell first, though it may not even belong to the bucket. + CellGroup* group = table->getCellGroups() + ((idx & sizeMask) >> 2); + cell = group->cells + (idx & 3); + Hash probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == KeyTraits::NullHash) { + if (cell->hash.compareExchangeStrong(probeHash, hash, turf::Relaxed)) { + TURF_TRACE(LeapFrog, 4, "[insert] reserved first cell", uptr(table), idx); + // There are no links to set. We're done. + return InsertResult_InsertedNew; + } else { + TURF_TRACE(LeapFrog, 5, "[insert] race to reserve first cell", uptr(table), idx); + // Fall through to check if it was the same hash... + } + } + if (probeHash == hash) { + TURF_TRACE(LeapFrog, 6, "[insert] found in first cell", uptr(table), idx); + return InsertResult_AlreadyFound; + } + + // Follow the link chain for this bucket. + ureg maxIdx = idx + sizeMask; + ureg linkLevel = 0; + turf::Atomic* prevLink; + for (;;) { + followLink: + prevLink = group->deltas + ((idx & 3) + linkLevel); + linkLevel = 4; + u8 probeDelta = prevLink->load(turf::Relaxed); + if (probeDelta) { + idx += probeDelta; + // Check the hash for this cell. + group = table->getCellGroups() + ((idx & sizeMask) >> 2); + cell = group->cells + (idx & 3); + probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == KeyTraits::NullHash) { + // Cell was linked, but hash is not visible yet. + // We could avoid this case (and guarantee it's visible) using acquire & release, but instead, + // just poll until it becomes visible. + TURF_TRACE(LeapFrog, 7, "[insert] race to read hash", uptr(table), idx); + do { + probeHash = cell->hash.load(turf::Acquire); + } while (probeHash == KeyTraits::NullHash); + } + TURF_ASSERT(((probeHash ^ hash) & sizeMask) == 0); // Only hashes in same bucket can be linked + if (probeHash == hash) { + TURF_TRACE(LeapFrog, 8, "[insert] found in probe chain", uptr(table), idx); + return InsertResult_AlreadyFound; + } + } else { + // Reached the end of the link chain for this bucket. + // Switch to linear probing until we reserve a new cell or find a late-arriving cell in the same bucket. + ureg prevLinkIdx = idx; + TURF_ASSERT(sreg(maxIdx - idx) >= 0); // Nobody would have linked an idx that's out of range. + ureg linearProbesRemaining = turf::util::min(maxIdx - idx, LinearSearchLimit); + while (linearProbesRemaining-- > 0) { + idx++; + group = table->getCellGroups() + ((idx & sizeMask) >> 2); + cell = group->cells + (idx & 3); + probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == KeyTraits::NullHash) { + // It's an empty cell. Try to reserve it. + if (cell->hash.compareExchangeStrong(probeHash, hash, turf::Relaxed)) { + // Success. We've reserved the cell. Link it to previous cell in same bucket. + TURF_TRACE(LeapFrog, 9, "[insert] reserved cell", uptr(table), idx); + TURF_ASSERT(probeDelta == 0); + u8 desiredDelta = idx - prevLinkIdx; +#if TURF_WITH_ASSERTS + probeDelta = prevLink->exchange(desiredDelta, turf::Relaxed); + TURF_ASSERT(probeDelta == 0 || probeDelta == desiredDelta); +#else + prevLink->store(desiredDelta, turf::Relaxed); +#endif + return InsertResult_InsertedNew; + } else { + TURF_TRACE(LeapFrog, 10, "[insert] race to reserve cell", uptr(table), idx); + // Fall through to check if it's the same hash... + } + } + Hash x = (probeHash ^ hash); + // Check for same hash. + if (!x) { + TURF_TRACE(LeapFrog, 11, "[insert] found outside probe chain", uptr(table), idx); + return InsertResult_AlreadyFound; + } + // Check for same bucket. + if ((x & sizeMask) == 0) { + TURF_TRACE(LeapFrog, 12, "[insert] found late-arriving cell in same bucket", uptr(table), idx); + // Attempt to set the link on behalf of the late-arriving cell. + // This is usually redundant, but if we don't attempt to set the late-arriving cell's link here, + // there's no guarantee that our own link chain will be well-formed by the time this function returns. + // (Indeed, subsequent lookups sometimes failed during testing, for this exact reason.) + u8 desiredDelta = idx - prevLinkIdx; +#if TURF_WITH_ASSERTS + probeDelta = prevLink->exchange(desiredDelta, turf::Relaxed); + TURF_ASSERT(probeDelta == 0 || probeDelta == desiredDelta); + if (probeDelta == 0) + TURF_TRACE(LeapFrog, 13, "[insert] set link on behalf of late-arriving cell", uptr(table), idx); +#else + prevLink->store(desiredDelta, turf::Relaxed); +#endif + goto followLink; // Try to follow link chain for the bucket again. + } + // Continue linear search... + } + // Table is too full to insert. + overflowIdx = idx + 1; + TURF_TRACE(LeapFrog, 14, "[insert] overflow", uptr(table), overflowIdx); + return InsertResult_Overflow; + } + } + } + + static void beginTableMigrationToSize(Map& map, Table* table, ureg nextTableSize) { + // Create new migration by DCLI. + TURF_TRACE(LeapFrog, 15, "[beginTableMigrationToSize] called", 0, 0); + SimpleJobCoordinator::Job* job = table->jobCoordinator.loadConsume(); + if (job) { + TURF_TRACE(LeapFrog, 16, "[beginTableMigrationToSize] new migration already exists", 0, 0); + } else { + turf::LockGuard guard(table->mutex); + job = table->jobCoordinator.loadConsume(); // Non-atomic would be sufficient, but that's OK. + if (job) { + TURF_TRACE(LeapFrog, 17, "[beginTableMigrationToSize] new migration already exists (double-checked)", 0, 0); + } else { + // Create new migration. + TableMigration* migration = TableMigration::create(map, 1); + migration->m_unitsRemaining.storeNonatomic(table->getNumMigrationUnits()); + migration->getSources()[0].table = table; + migration->getSources()[0].sourceIndex.storeNonatomic(0); + migration->m_destination = Table::create(nextTableSize); + // Publish the new migration. + table->jobCoordinator.storeRelease(migration); + } + } + } + + static void beginTableMigration(Map& map, Table* table, ureg overflowIdx) { + // Estimate number of cells in use based on a small sample. + ureg sizeMask = table->sizeMask; + ureg idx = overflowIdx - CellsInUseSample; + ureg inUseCells = 0; + for (ureg linearProbesRemaining = CellsInUseSample; linearProbesRemaining > 0; linearProbesRemaining--) { + CellGroup* group = table->getCellGroups() + ((idx & sizeMask) >> 2); + Cell* cell = group->cells + (idx & 3); + Value value = cell->value.load(turf::Relaxed); + if (value == Value(ValueTraits::Redirect)) { + // Another thread kicked off the jobCoordinator. The caller will participate upon return. + TURF_TRACE(LeapFrog, 18, "[beginTableMigration] redirected while determining table size", 0, 0); + return; + } + if (value != Value(ValueTraits::NullValue)) + inUseCells++; + idx++; + } + float inUseRatio = float(inUseCells) / CellsInUseSample; + float estimatedInUse = (sizeMask + 1) * inUseRatio; + ureg nextTableSize = turf::util::roundUpPowerOf2(ureg(estimatedInUse * 2)); + beginTableMigrationToSize(map, table, nextTableSize); + } +}; // LeapFrog + +template +bool LeapFrog::TableMigration::migrateRange(Table* srcTable, ureg startIdx) { + ureg srcSizeMask = srcTable->sizeMask; + ureg endIdx = turf::util::min(startIdx + TableMigrationUnitSize, srcSizeMask + 1); + // Iterate over source range. + for (ureg srcIdx = startIdx; srcIdx < endIdx; srcIdx++) { + CellGroup* srcGroup = srcTable->getCellGroups() + ((srcIdx & srcSizeMask) >> 2); + Cell* srcCell = srcGroup->cells + (srcIdx & 3); + Hash srcHash; + Value srcValue; + // Fetch the srcHash and srcValue. + for (;;) { + srcHash = srcCell->hash.load(turf::Relaxed); + if (srcHash == KeyTraits::NullHash) { + // An unused cell. Try to put a Redirect marker in its value. + srcValue = srcCell->value.compareExchange(Value(ValueTraits::NullValue), Value(ValueTraits::Redirect), turf::Relaxed); + if (srcValue == Value(ValueTraits::Redirect)) { + // srcValue is already marked Redirect due to previous incomplete migration. + TURF_TRACE(LeapFrog, 19, "[migrateRange] empty cell already redirected", uptr(srcTable), srcIdx); + break; + } + if (srcValue == Value(ValueTraits::NullValue)) + break; // Redirect has been placed. Break inner loop, continue outer loop. + TURF_TRACE(LeapFrog, 20, "[migrateRange] race to insert key", uptr(srcTable), srcIdx); + // Otherwise, somebody just claimed the cell. Read srcHash again... + } else { + // Check for deleted/uninitialized value. + srcValue = srcCell->value.load(turf::Relaxed); + if (srcValue == Value(ValueTraits::NullValue)) { + // Try to put a Redirect marker. + if (srcCell->value.compareExchangeStrong(srcValue, Value(ValueTraits::Redirect), turf::Relaxed)) + break; // Redirect has been placed. Break inner loop, continue outer loop. + TURF_TRACE(LeapFrog, 21, "[migrateRange] race to insert value", uptr(srcTable), srcIdx); + if (srcValue == Value(ValueTraits::Redirect)) { + // FIXME: I don't think this will happen. Investigate & change to assert + TURF_TRACE(LeapFrog, 22, "[migrateRange] race inserted Redirect", uptr(srcTable), srcIdx); + break; + } + } else if (srcValue == Value(ValueTraits::Redirect)) { + // srcValue is already marked Redirect due to previous incomplete migration. + TURF_TRACE(LeapFrog, 23, "[migrateRange] in-use cell already redirected", uptr(srcTable), srcIdx); + break; + } + + // We've got a key/value pair to migrate. + // Reserve a destination cell in the destination. + TURF_ASSERT(srcHash != KeyTraits::NullHash); + TURF_ASSERT(srcValue != Value(ValueTraits::NullValue)); + TURF_ASSERT(srcValue != Value(ValueTraits::Redirect)); + Cell* dstCell; + ureg overflowIdx; + InsertResult result = insert(srcHash, m_destination, dstCell, overflowIdx); + // During migration, a hash can only exist in one place among all the source tables, + // and it is only migrated by one thread. Therefore, the hash will never already exist + // in the destination table: + TURF_ASSERT(result != InsertResult_AlreadyFound); + if (result == InsertResult_Overflow) { + // Destination overflow. + // This can happen for several reasons. For example, the source table could have + // existed of all deleted cells when it overflowed, resulting in a small destination + // table size, but then another thread could re-insert all the same hashes + // before the migration completed. + // Caller will cancel the current migration and begin a new one. + return false; + } + // Migrate the old value to the new cell. + for (;;) { + // Copy srcValue to the destination. + dstCell->value.store(srcValue, turf::Relaxed); + // Try to place a Redirect marker in srcValue. + Value doubleCheckedSrcValue = srcCell->value.compareExchange(srcValue, Value(ValueTraits::Redirect), turf::Relaxed); + TURF_ASSERT(doubleCheckedSrcValue != Value(ValueTraits::Redirect)); // Only one thread can redirect a cell at a time. + if (doubleCheckedSrcValue == srcValue) { + // No racing writes to the src. We've successfully placed the Redirect marker. + // srcValue was non-NULL when we decided to migrate it, but it may have changed to NULL + // by a late-arriving erase. + if (srcValue == Value(ValueTraits::NullValue)) + TURF_TRACE(LeapFrog, 24, "[migrateRange] racing update was erase", uptr(srcTable), srcIdx); + break; + } + // There was a late-arriving write (or erase) to the src. Migrate the new value and try again. + TURF_TRACE(LeapFrog, 25, "[migrateRange] race to update migrated value", uptr(srcTable), srcIdx); + srcValue = doubleCheckedSrcValue; + } + // Cell successfully migrated. Proceed to next source cell. + break; + } + } + } + // Range has been migrated successfully. + return true; +} + +template +void LeapFrog::TableMigration::run() { + // Conditionally increment the shared # of workers. + ureg probeStatus = m_workerStatus.load(turf::Relaxed); + do { + if (probeStatus & 1) { + // End flag is already set, so do nothing. + TURF_TRACE(LeapFrog, 26, "[TableMigration::run] already ended", uptr(this), 0); + return; + } + } while (!m_workerStatus.compareExchangeWeak(probeStatus, probeStatus + 2, turf::Relaxed, turf::Relaxed)); + // # of workers has been incremented, and the end flag is clear. + TURF_ASSERT((probeStatus & 1) == 0); + + // Iterate over all source tables. + for (ureg s = 0; s < m_numSources; s++) { + Source& source = getSources()[s]; + // Loop over all migration units in this source table. + for (;;) { + if (m_workerStatus.load(turf::Relaxed) & 1) { + TURF_TRACE(LeapFrog, 27, "[TableMigration::run] detected end flag set", uptr(this), 0); + goto endMigration; + } + ureg startIdx = source.sourceIndex.fetchAdd(TableMigrationUnitSize, turf::Relaxed); + if (startIdx >= source.table->sizeMask + 1) + break; // No more migration units in this table. Try next source table. + bool overflowed = !migrateRange(source.table, startIdx); + if (overflowed) { + // *** FAILED MIGRATION *** + // TableMigration failed due to destination table overflow. + // No other thread can declare the migration successful at this point, because *this* unit will never complete, hence m_unitsRemaining won't reach zero. + // However, multiple threads can independently detect a failed migration at the same time. + TURF_TRACE(LeapFrog, 28, "[TableMigration::run] destination overflow", uptr(source.table), uptr(startIdx)); + // The reason we store overflowed in a shared variable is because we can must flush all the worker threads before + // we can safely deal with the overflow. Therefore, the thread that detects the failure is often different from the thread + // that deals with it. + bool oldOverflowed = m_overflowed.exchange(overflowed, turf::Relaxed); + if (oldOverflowed) + TURF_TRACE(LeapFrog, 29, "[TableMigration::run] race to set m_overflowed", uptr(overflowed), uptr(oldOverflowed)); + m_workerStatus.fetchOr(1, turf::Relaxed); + goto endMigration; + } + sreg prevRemaining = m_unitsRemaining.fetchSub(1, turf::Relaxed); + TURF_ASSERT(prevRemaining > 0); + if (prevRemaining == 1) { + // *** SUCCESSFUL MIGRATION *** + // That was the last chunk to migrate. + m_workerStatus.fetchOr(1, turf::Relaxed); + goto endMigration; + } + } + } + TURF_TRACE(LeapFrog, 30, "[TableMigration::run] out of migration units", uptr(this), 0); + +endMigration: + // Decrement the shared # of workers. + probeStatus = m_workerStatus.fetchSub(2, turf::AcquireRelease); // AcquireRelease makes all previous writes visible to the last worker thread. + if (probeStatus >= 4) { + // There are other workers remaining. Return here so that only the very last worker will proceed. + TURF_TRACE(LeapFrog, 31, "[TableMigration::run] not the last worker", uptr(this), uptr(probeStatus)); + return; + } + + // We're the very last worker thread. + // Perform the appropriate post-migration step depending on whether the migration succeeded or failed. + TURF_ASSERT(probeStatus == 3); + bool overflowed = m_overflowed.loadNonatomic(); // No racing writes at this point + if (!overflowed) { + // The migration succeeded. This is the most likely outcome. Publish the new subtree. + m_map.publishTableMigration(this); + // End the jobCoodinator. + getSources()[0].table->jobCoordinator.end(); + } else { + // The migration failed due to the overflow of the destination table. + Table* origTable = getSources()[0].table; + turf::LockGuard guard(origTable->mutex); + SimpleJobCoordinator::Job* checkedJob = origTable->jobCoordinator.loadConsume(); + if (checkedJob != this) { + TURF_TRACE(LeapFrog, 32, "[TableMigration::run] a new TableMigration was already started", uptr(origTable), uptr(checkedJob)); + } else { + TableMigration* migration = TableMigration::create(m_map, m_numSources + 1); + // Double the destination table size. + migration->m_destination = Table::create((m_destination->sizeMask + 1) * 2); + // Transfer source tables to the new migration. + for (ureg i = 0; i < m_numSources; i++) { + migration->getSources()[i].table = getSources()[i].table; + getSources()[i].table = NULL; + migration->getSources()[i].sourceIndex.storeNonatomic(0); + } + migration->getSources()[m_numSources].table = m_destination; + migration->getSources()[m_numSources].sourceIndex.storeNonatomic(0); + // Calculate total number of migration units to move. + ureg unitsRemaining = 0; + for (ureg s = 0; s < migration->m_numSources; s++) + unitsRemaining += migration->getSources()[s].table->getNumMigrationUnits(); + migration->m_unitsRemaining.storeNonatomic(unitsRemaining); + // Publish the new migration. + origTable->jobCoordinator.storeRelease(migration); + } + } + + // We're done with this TableMigration. Queue it for GC. + DefaultQSBR.enqueue(&TableMigration::destroy, this); +} + +} // namespace details +} // namespace junction + +#endif // JUNCTION_DETAILS_LEAPFROG_H diff --git a/junction/details/Linear.cpp b/junction/details/Linear.cpp new file mode 100644 index 0000000..d731567 --- /dev/null +++ b/junction/details/Linear.cpp @@ -0,0 +1,46 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include + +namespace junction { +namespace details { + +TURF_TRACE_DEFINE_BEGIN(Linear, 22) // autogenerated by TidySource.py +TURF_TRACE_DEFINE("[find] called") +TURF_TRACE_DEFINE("[find] found existing cell") +TURF_TRACE_DEFINE("[insert] called") +TURF_TRACE_DEFINE("[insert] found existing cell") +TURF_TRACE_DEFINE("[insert] ran out of cellsRemaining") +TURF_TRACE_DEFINE("[insert] reserved cell") +TURF_TRACE_DEFINE("[insert] detected race to reserve cell") +TURF_TRACE_DEFINE("[insert] race reserved same hash") +TURF_TRACE_DEFINE("[beginTableMigration] called") +TURF_TRACE_DEFINE("[beginTableMigration] new migration already exists") +TURF_TRACE_DEFINE("[beginTableMigration] new migration already exists (double-checked)") +TURF_TRACE_DEFINE("[table] restarting valuesRemaining CAS loop") +TURF_TRACE_DEFINE("[table] valuesRemaining CAS failed") +TURF_TRACE_DEFINE("[migrateRange] empty cell already redirected") +TURF_TRACE_DEFINE("[migrateRange] race to insert key") +TURF_TRACE_DEFINE("[migrateRange] race to insert value") +TURF_TRACE_DEFINE("[migrateRange] racing update was erase") +TURF_TRACE_DEFINE("[migrateRange] race to update migrated value") +TURF_TRACE_DEFINE("[TableMigration::run] already ended") +TURF_TRACE_DEFINE("[TableMigration::run] detected end flag set") +TURF_TRACE_DEFINE("[TableMigration::run] out of migration units") +TURF_TRACE_DEFINE("[TableMigration::run] not the last worker") +TURF_TRACE_DEFINE_END(Linear, 22) + +} // namespace details +} // namespace junction diff --git a/junction/details/Linear.h b/junction/details/Linear.h new file mode 100644 index 0000000..f466cb0 --- /dev/null +++ b/junction/details/Linear.h @@ -0,0 +1,375 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_DETAILS_LINEAR_H +#define JUNCTION_DETAILS_LINEAR_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace junction { +namespace details { + +TURF_TRACE_DECLARE(Linear, 22) + +template +struct Linear { + typedef typename Map::Hash Hash; + typedef typename Map::Value Value; + typedef typename Map::KeyTraits KeyTraits; + typedef typename Map::ValueTraits ValueTraits; + + static const ureg InitialSize = 8; + static const ureg TableMigrationUnitSize = 32; + static const ureg LinearSearchLimit = 128; + static const ureg CellsInUseSample = LinearSearchLimit; + TURF_STATIC_ASSERT(LinearSearchLimit > 0 && LinearSearchLimit < 256); // Must fit in CellGroup::links + TURF_STATIC_ASSERT(CellsInUseSample > 0 && CellsInUseSample <= LinearSearchLimit); // Limit sample to failed search chain + + struct Cell { + turf::Atomic hash; + turf::Atomic value; + }; + + struct Table { + const ureg sizeMask; // a power of two minus one + const ureg limitNumValues; + turf::Atomic cellsRemaining; + turf::Atomic valuesRemaining; + turf::Mutex mutex; // to DCLI the TableMigration (stored in the jobCoordinator) + SimpleJobCoordinator jobCoordinator; // makes all blocked threads participate in the migration + + Table(ureg sizeMask, ureg limitNumValues) : sizeMask(sizeMask), limitNumValues(limitNumValues), + cellsRemaining(limitNumValues), valuesRemaining(limitNumValues) { + } + + static Table* create(ureg tableSize, ureg limitNumValues) { + TURF_ASSERT(turf::util::isPowerOf2(tableSize)); + Table* table = (Table*) TURF_HEAP.alloc(sizeof(Table) + sizeof(Cell) * tableSize); + new(table) Table(tableSize - 1, limitNumValues); + for (ureg j = 0; j < tableSize; j++) { + table->getCells()[j].hash.storeNonatomic(KeyTraits::NullHash); + table->getCells()[j].value.storeNonatomic(Value(ValueTraits::NullValue)); + } + return table; + } + + void destroy() { + this->Table::~Table(); + TURF_HEAP.free(this); + } + + Cell* getCells() const { + return (Cell*) (this + 1); + } + + ureg getNumMigrationUnits() const { + return sizeMask / TableMigrationUnitSize + 1; + } + }; + + class TableMigration : public SimpleJobCoordinator::Job { + public: + Map& m_map; + Table* m_source; + turf::Atomic m_sourceIndex; + Table* m_destination; + turf::Atomic m_workerStatus; // number of workers + end flag + turf::Atomic m_unitsRemaining; + + TableMigration(Map& map) : m_map(map), m_sourceIndex(0), m_workerStatus(0), m_unitsRemaining(0) { + // Caller is responsible for filling in source & destination + } + + virtual ~TableMigration() TURF_OVERRIDE { + // Destroy source table. + m_source->destroy(); + } + + void destroy() { + delete this; + } + + bool migrateRange(ureg startIdx); + virtual void run() TURF_OVERRIDE; + }; + + static Cell* find(Hash hash, Table* table) { + TURF_TRACE(Linear, 0, "[find] called", uptr(table), hash); + TURF_ASSERT(table); + TURF_ASSERT(hash != KeyTraits::NullHash); + ureg sizeMask = table->sizeMask; + for (ureg idx = hash;; idx++) { + idx &= sizeMask; + Cell* cell = table->getCells() + idx; + // Load the hash that was there. + uptr probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == hash) { + TURF_TRACE(Linear, 1, "[find] found existing cell", uptr(table), idx); + return cell; + } else if (probeHash == KeyTraits::NullHash) { + return NULL; + } + } + } + + // FIXME: Possible optimization: Dedicated insert for migration? It wouldn't check for InsertResult_AlreadyFound. + enum InsertResult { + InsertResult_AlreadyFound, + InsertResult_InsertedNew, + InsertResult_Overflow + }; + static InsertResult insert(Hash hash, Table* table, Cell*& cell) { + TURF_TRACE(Linear, 2, "[insert] called", uptr(table), hash); + TURF_ASSERT(table); + TURF_ASSERT(hash != KeyTraits::NullHash); + ureg sizeMask = table->sizeMask; + + for (ureg idx = hash;; idx++) { + idx &= sizeMask; + cell = table->getCells() + idx; + // Load the existing hash. + uptr probeHash = cell->hash.load(turf::Relaxed); + if (probeHash == hash) { + TURF_TRACE(Linear, 3, "[insert] found existing cell", uptr(table), idx); + return InsertResult_AlreadyFound; // Key found in table. Return the existing cell. + } + if (probeHash == KeyTraits::NullHash) { + // It's an empty cell. Try to reserve it. + // But first, decrement cellsRemaining to ensure we have permission to create new getCells(). + s32 prevCellsRemaining = table->cellsRemaining.fetchSub(1, turf::Relaxed); + if (prevCellsRemaining <= 0) { + // Table is overpopulated. + TURF_TRACE(Linear, 4, "[insert] ran out of cellsRemaining", prevCellsRemaining, 0); + table->cellsRemaining.fetchAdd(1, turf::Relaxed); // Undo cellsRemaining decrement + return InsertResult_Overflow; + } + // Try to reserve this cell. + uptr prevHash = cell->hash.compareExchange(KeyTraits::NullHash, hash, turf::Relaxed); + if (prevHash == KeyTraits::NullHash) { + // Success. We reserved a new cell. + TURF_TRACE(Linear, 5, "[insert] reserved cell", prevCellsRemaining, idx); + return InsertResult_InsertedNew; + } + // There was a race and another thread reserved that cell from under us. + TURF_TRACE(Linear, 6, "[insert] detected race to reserve cell", ureg(hash), idx); + table->cellsRemaining.fetchAdd(1, turf::Relaxed); // Undo cellsRemaining decrement + if (prevHash == hash) { + TURF_TRACE(Linear, 7, "[insert] race reserved same hash", ureg(hash), idx); + return InsertResult_AlreadyFound; // They inserted the same key. Return the existing cell. + } + } + // Try again in the next cell. + } + } + + static void beginTableMigration(Map& map, Table* table) { + // Create new migration by DCLI. + TURF_TRACE(Linear, 8, "[beginTableMigration] called", 0, 0); + SimpleJobCoordinator::Job* job = table->jobCoordinator.loadConsume(); + if (job) { + TURF_TRACE(Linear, 9, "[beginTableMigration] new migration already exists", 0, 0); + } else { + turf::LockGuard guard(table->mutex); + job = table->jobCoordinator.loadConsume(); // Non-atomic would be sufficient, but that's OK. + if (job) { + TURF_TRACE(Linear, 10, "[beginTableMigration] new migration already exists (double-checked)", 0, 0); + } else { + // Determine new migration size and cap the number of values that can be added concurrent to the migration. + sreg oldValuesLimit = table->limitNumValues; + sreg oldValuesRemaining = table->valuesRemaining.load(turf::Relaxed); + sreg oldValuesInUse = oldValuesLimit - oldValuesRemaining; + calculateNextTableSize: + sreg nextTableSize = turf::util::roundUpPowerOf2(oldValuesInUse * 2); + sreg nextLimitNumValues = nextTableSize * 3 / 4; + if (nextLimitNumValues < oldValuesLimit) { + // Set the new limitNumValues on the *current* table. + // This prevents other threads, while the migration is in progress, from concurrently + // re-inserting more values than the new table can hold. + // To set the new limitNumValues on the current table in an atomic fashion, + // we update its valuesRemaining via CAS loop: + for(;;) { + // We must recalculate desiredValuesRemaining on each iteration of the CAS loop + oldValuesInUse = oldValuesLimit - oldValuesRemaining; + sreg desiredValuesRemaining = nextLimitNumValues - oldValuesInUse; + if (desiredValuesRemaining < 0) { + TURF_TRACE(Linear, 11, "[table] restarting valuesRemaining CAS loop", nextLimitNumValues, desiredValuesRemaining); + // Must recalculate nextTableSize. Goto, baby! + goto calculateNextTableSize; + } + if (table->valuesRemaining.compareExchangeWeak(oldValuesRemaining, desiredValuesRemaining, turf::Relaxed, turf::Relaxed)) + break; // Success! + // CAS failed because table->valuesRemaining was modified by another thread. + // An updated value has been reloaded into oldValuesRemaining (modified by reference). + // Recalculate desiredValuesRemaining to account for the updated value, and try again. + TURF_TRACE(Linear, 12, "[table] valuesRemaining CAS failed", oldValuesRemaining, desiredValuesRemaining); + } + } + // Now we are assured that the new table will not become overpopulated during the migration process. + // Create new migration. + TableMigration* migration = new TableMigration(map); + migration->m_source = table; + migration->m_destination = Table::create(nextTableSize, nextLimitNumValues); + migration->m_unitsRemaining.storeNonatomic(table->getNumMigrationUnits()); + // Publish the new migration. + table->jobCoordinator.storeRelease(migration); + } + } + } +}; // Linear + +template +bool Linear::TableMigration::migrateRange(ureg startIdx) { + ureg srcSizeMask = m_source->sizeMask; + ureg endIdx = turf::util::min(startIdx + TableMigrationUnitSize, srcSizeMask + 1); + sreg valuesMigrated = 0; + // Iterate over source range. + for (ureg srcIdx = startIdx; srcIdx < endIdx; srcIdx++) { + Cell* srcCell = m_source->getCells() + (srcIdx & srcSizeMask); + Hash srcHash; + Value srcValue; + // Fetch the srcHash and srcValue. + for (;;) { + srcHash = srcCell->hash.load(turf::Relaxed); + if (srcHash == KeyTraits::NullHash) { + // An unused cell. Try to put a Redirect marker in its value. + srcValue = srcCell->value.compareExchange(Value(ValueTraits::NullValue), Value(ValueTraits::Redirect), turf::Relaxed); + if (srcValue == Value(ValueTraits::Redirect)) { + // srcValue is already marked Redirect due to previous incomplete migration. + TURF_TRACE(Linear, 13, "[migrateRange] empty cell already redirected", uptr(m_source), srcIdx); + break; + } + if (srcValue == Value(ValueTraits::NullValue)) + break; // Redirect has been placed. Break inner loop, continue outer loop. + TURF_TRACE(Linear, 14, "[migrateRange] race to insert key", uptr(m_source), srcIdx); + // Otherwise, somebody just claimed the cell. Read srcHash again... + } else { + // Check for deleted/uninitialized value. + srcValue = srcCell->value.load(turf::Relaxed); + if (srcValue == Value(ValueTraits::NullValue)) { + // Try to put a Redirect marker. + if (srcCell->value.compareExchangeStrong(srcValue, Value(ValueTraits::Redirect), turf::Relaxed)) + break; // Redirect has been placed. Break inner loop, continue outer loop. + TURF_TRACE(Linear, 15, "[migrateRange] race to insert value", uptr(m_source), srcIdx); + } + + // We've got a key/value pair to migrate. + // Reserve a destination cell in the destination. + TURF_ASSERT(srcHash != KeyTraits::NullHash); + TURF_ASSERT(srcValue != Value(ValueTraits::NullValue)); + TURF_ASSERT(srcValue != Value(ValueTraits::Redirect)); // Incomplete/concurrent migrations are impossible. + Cell* dstCell; + InsertResult result = insert(srcHash, m_destination, dstCell); + // During migration, a hash can only exist in one place among all the source tables, + // and it is only migrated by one thread. Therefore, the hash will never already exist + // in the destination table: + TURF_ASSERT(result != InsertResult_AlreadyFound); + TURF_ASSERT(result != InsertResult_Overflow); + // Migrate the old value to the new cell. + for (;;) { + // Copy srcValue to the destination. + dstCell->value.store(srcValue, turf::Relaxed); + // Try to place a Redirect marker in srcValue. + Value doubleCheckedSrcValue = srcCell->value.compareExchange(srcValue, Value(ValueTraits::Redirect), turf::Relaxed); + TURF_ASSERT(doubleCheckedSrcValue != Value(ValueTraits::Redirect)); // Only one thread can redirect a cell at a time. + if (doubleCheckedSrcValue == srcValue) { + // No racing writes to the src. We've successfully placed the Redirect marker. + // srcValue was non-NULL when we decided to migrate it, but it may have changed to NULL + // by a late-arriving erase. + if (srcValue == Value(ValueTraits::NullValue)) + TURF_TRACE(Linear, 16, "[migrateRange] racing update was erase", uptr(m_source), srcIdx); + else + valuesMigrated++; + break; + } + // There was a late-arriving write (or erase) to the src. Migrate the new value and try again. + TURF_TRACE(Linear, 17, "[migrateRange] race to update migrated value", uptr(m_source), srcIdx); + srcValue = doubleCheckedSrcValue; + } + // Cell successfully migrated. Proceed to next source cell. + break; + } + } + } + sreg prevValuesRemaining = m_destination->valuesRemaining.fetchSub(valuesMigrated, turf::Relaxed); + TURF_ASSERT(valuesMigrated <= prevValuesRemaining); + TURF_UNUSED(prevValuesRemaining); + // Range has been migrated successfully. + return true; +} + +template +void Linear::TableMigration::run() { + // Conditionally increment the shared # of workers. + ureg probeStatus = m_workerStatus.load(turf::Relaxed); + do { + if (probeStatus & 1) { + // End flag is already set, so do nothing. + TURF_TRACE(Linear, 18, "[TableMigration::run] already ended", uptr(this), 0); + return; + } + } while (!m_workerStatus.compareExchangeWeak(probeStatus, probeStatus + 2, turf::Relaxed, turf::Relaxed)); + // # of workers has been incremented, and the end flag is clear. + TURF_ASSERT((probeStatus & 1) == 0); + + // Loop over all migration units in the source table. + for (;;) { + if (m_workerStatus.load(turf::Relaxed) & 1) { + TURF_TRACE(Linear, 19, "[TableMigration::run] detected end flag set", uptr(this), 0); + goto endMigration; + } + ureg startIdx = m_sourceIndex.fetchAdd(TableMigrationUnitSize, turf::Relaxed); + if (startIdx >= m_source->sizeMask + 1) + break; // No more migration units. + migrateRange(startIdx); + sreg prevRemaining = m_unitsRemaining.fetchSub(1, turf::Relaxed); + TURF_ASSERT(prevRemaining > 0); + if (prevRemaining == 1) { + // That was the last chunk to migrate. + m_workerStatus.fetchOr(1, turf::Relaxed); + goto endMigration; + } + } + TURF_TRACE(Linear, 20, "[TableMigration::run] out of migration units", uptr(this), 0); + +endMigration: + // Decrement the shared # of workers. + probeStatus = m_workerStatus.fetchSub(2, turf::AcquireRelease); // AcquireRelease makes all previous writes visible to the last worker thread. + if (probeStatus >= 4) { + // There are other workers remaining. Return here so that only the very last worker will proceed. + TURF_TRACE(Linear, 21, "[TableMigration::run] not the last worker", uptr(this), uptr(probeStatus)); + return; + } + + // We're the very last worker thread. + // Publish the new subtree. + TURF_ASSERT(probeStatus == 3); + m_map.publishTableMigration(this); + // End the jobCoodinator. + m_source->jobCoordinator.end(); + + // We're done with this TableMigration. Queue it for GC. + DefaultQSBR.enqueue(&TableMigration::destroy, this); +} + +} // namespace details +} // namespace junction + +#endif // JUNCTION_DETAILS_LINEAR_H diff --git a/junction/extra/MapAdapter.h b/junction/extra/MapAdapter.h new file mode 100644 index 0000000..9897d94 --- /dev/null +++ b/junction/extra/MapAdapter.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_MAPADAPTER_H +#define JUNCTION_EXTRA_MAPADAPTER_H + +#include + +#ifndef JUNCTION_IMPL_MAPADAPTER_PATH +#define JUNCTION_IMPL_MAPADAPTER_PATH "junction/extra/impl/MapAdapter_Grampa.h" +#endif + +#include JUNCTION_IMPL_MAPADAPTER_PATH + +#endif // JUNCTION_EXTRA_MAPADAPTER_H diff --git a/junction/extra/MemHook_NBDS.cpp b/junction/extra/MemHook_NBDS.cpp new file mode 100644 index 0000000..c736748 --- /dev/null +++ b/junction/extra/MemHook_NBDS.cpp @@ -0,0 +1,29 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include + +#if JUNCTION_WITH_NBDS && NBDS_USE_TURF_HEAP +extern "C" { +void mem_init (void) { +} + +void *nbd_malloc (size_t n) { + return TURF_HEAP.alloc(n); +} + +void nbd_free (void *x) { + TURF_HEAP.free(x); +} +} // extern "C" +#endif // JUNCTION_WITH_NBDS && NBDS_USE_TURF_HEAP diff --git a/junction/extra/MemHook_TBB.cpp b/junction/extra/MemHook_TBB.cpp new file mode 100644 index 0000000..46dd76b --- /dev/null +++ b/junction/extra/MemHook_TBB.cpp @@ -0,0 +1,32 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include + +#if JUNCTION_WITH_TBB && TBB_USE_TURF_HEAP +void* tbbWrap_malloc(size_t size) { + return TURF_HEAP.alloc(size); +} + +void tbbWrap_free(void* ptr) { + TURF_HEAP.free(ptr); +} + +void* tbbWrap_padded_allocate(size_t size, size_t alignment) { + return TURF_HEAP.allocAligned(size, alignment); +} + +void tbbWrap_padded_free(void* ptr) { + TURF_HEAP.free(ptr); +} +#endif // JUNCTION_WITH_TBB && TBB_USE_TURF_HEAP diff --git a/junction/extra/impl/MapAdapter_CDS_Cuckoo.h b/junction/extra/impl/MapAdapter_CDS_Cuckoo.h new file mode 100644 index 0000000..bb22a67 --- /dev/null +++ b/junction/extra/impl/MapAdapter_CDS_Cuckoo.h @@ -0,0 +1,112 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_CDS_CUCKOO_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_CDS_CUCKOO_H + +#include + +#if !JUNCTION_WITH_CDS +#error "You must configure with JUNCTION_WITH_CDS=1!" +#endif + +#include +#include +#include +#include // memcpy required by cuckoo_map.h +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "CDS CuckooMap"; + + cds::gc::HP *m_hpGC; + + MapAdapter(ureg) { + cds::Initialize(); + m_hpGC = new cds::gc::HP; + } + + ~MapAdapter() { + delete m_hpGC; + cds::Terminate(); + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + cds::threading::Manager::attachThread(); + } + + void unregisterThread() { + cds::threading::Manager::detachThread(); + } + + void update() { + } + }; + + class Map { + private: + struct Hash1 { + size_t operator()(u32 s) const { + return junction::hash(s); + } + }; + + struct Hash2 { + size_t operator()(u32 s) const { + return junction::hash(s + 0x9e3779b9); + } + }; + + struct Traits : cds::container::cuckoo::traits { + typedef std::equal_to equal_to; + typedef cds::opt::hash_tuple hash; + }; + + cds::container::CuckooMap m_map; + + public: + Map(ureg capacity) : m_map() { + } + + void insert(u32 key, void* value) { + m_map.insert(key, value); + } + + void* get(u32 key) { + void* result = NULL; + m_map.find(key, [&result](std::pair& item){ result = item.second; }); + return result; + } + + void erase(u32 key) { + m_map.erase(key); + } + }; + + static ureg getInitialCapacity(ureg maxPopulation) { + return maxPopulation / 4; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_CDS_CUCKOO_H diff --git a/junction/extra/impl/MapAdapter_CDS_Michael.h b/junction/extra/impl/MapAdapter_CDS_Michael.h new file mode 100644 index 0000000..ce600f3 --- /dev/null +++ b/junction/extra/impl/MapAdapter_CDS_Michael.h @@ -0,0 +1,115 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_CDS_MICHAEL_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_CDS_MICHAEL_H + +#include + +#if !JUNCTION_WITH_CDS +#error "You must configure with JUNCTION_WITH_CDS=1!" +#endif + +#include +#include +#include +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "CDS MichaelKVList"; + + cds::gc::HP *m_hpGC; + + MapAdapter(ureg) { + cds::Initialize(); + m_hpGC = new cds::gc::HP; + } + + ~MapAdapter() { + delete m_hpGC; + cds::Terminate(); + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + cds::threading::Manager::attachThread(); + } + + void unregisterThread() { + cds::threading::Manager::detachThread(); + } + + void update() { + } + }; + + class Map { + private: + // List traits based on std::less predicate + struct ListTraits : public cds::container::michael_list::traits + { + typedef std::less less; + }; + + // Ordered list + typedef cds::container::MichaelKVList< cds::gc::HP, u32, void*, ListTraits> OrderedList; + + // Map traits + struct MapTraits : public cds::container::michael_map::traits + { + struct hash { + size_t operator()( u32 i ) const + { + return cds::opt::v::hash()( i ); + } + }; + }; + + cds::container::MichaelHashMap m_map; + + public: + Map(ureg capacity) : m_map(capacity, 1) { + } + + void insert(u32 key, void* value) { + m_map.insert(key, value); + } + + void* get(u32 key) { + void* result = NULL; + m_map.find(key, [&result](std::pair& item){ result = item.second; }); + return result; + } + + void erase(u32 key) { + m_map.erase(key); + } + }; + + static ureg getInitialCapacity(ureg maxPopulation) { + return maxPopulation / 4; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_CDS_MICHAEL_H diff --git a/junction/extra/impl/MapAdapter_Folly.h b/junction/extra/impl/MapAdapter_Folly.h new file mode 100644 index 0000000..02d54a6 --- /dev/null +++ b/junction/extra/impl/MapAdapter_Folly.h @@ -0,0 +1,79 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_FOLLY_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_FOLLY_H + +#include + +#if !JUNCTION_WITH_FOLLY +#error "You must configure with JUNCTION_WITH_FOLLY=1!" +#endif + +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Folly AtomicHashMap"; + + MapAdapter(ureg) { + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + } + + void unregisterThread() { + } + + void update() { + } + }; + + class Map { + private: + folly::AtomicHashMap m_map; + + public: + Map(ureg capacity) : m_map(capacity) { + } + + void insert(u32 key, void* value) { + m_map.insert(std::make_pair(key, value)); + } + + void* get(u32 key) { + auto ret = m_map.find(key); + return ret != m_map.end() ? ret->second : NULL; + } + + void erase(u32 key) { + m_map.erase(key); + } + }; + + static ureg getInitialCapacity(ureg maxPopulation) { + return maxPopulation / 4; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_FOLLY_H diff --git a/junction/extra/impl/MapAdapter_Grampa.h b/junction/extra/impl/MapAdapter_Grampa.h new file mode 100644 index 0000000..185fa94 --- /dev/null +++ b/junction/extra/impl/MapAdapter_Grampa.h @@ -0,0 +1,62 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_GRAMPA_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_GRAMPA_H + +#include +#include +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Junction Grampa map"; + + MapAdapter(ureg) { + } + + class ThreadContext { + private: + QSBR::Context m_qsbrContext; + + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + m_qsbrContext = DefaultQSBR.createContext(); + } + + void unregisterThread() { + DefaultQSBR.destroyContext(m_qsbrContext); + } + + void update() { + DefaultQSBR.update(m_qsbrContext); + } + }; + + typedef ConcurrentMap_Grampa Map; + + static ureg getInitialCapacity(ureg maxPopulation) { + return turf::util::roundUpPowerOf2(maxPopulation / 4); + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_GRAMPA_H diff --git a/junction/extra/impl/MapAdapter_LeapFrog.h b/junction/extra/impl/MapAdapter_LeapFrog.h new file mode 100644 index 0000000..02b09ca --- /dev/null +++ b/junction/extra/impl/MapAdapter_LeapFrog.h @@ -0,0 +1,62 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_LEAPFROG_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_LEAPFROG_H + +#include +#include +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Junction LeapFrog map"; + + MapAdapter(ureg) { + } + + class ThreadContext { + private: + QSBR::Context m_qsbrContext; + + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + m_qsbrContext = DefaultQSBR.createContext(); + } + + void unregisterThread() { + DefaultQSBR.destroyContext(m_qsbrContext); + } + + void update() { + DefaultQSBR.update(m_qsbrContext); + } + }; + + typedef ConcurrentMap_LeapFrog Map; + + static ureg getInitialCapacity(ureg maxPopulation) { + return turf::util::roundUpPowerOf2(maxPopulation / 4); + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_LEAPFROG_H diff --git a/junction/extra/impl/MapAdapter_Linear.h b/junction/extra/impl/MapAdapter_Linear.h new file mode 100644 index 0000000..525eb74 --- /dev/null +++ b/junction/extra/impl/MapAdapter_Linear.h @@ -0,0 +1,62 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_H + +#include +#include +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Junction Linear map"; + + MapAdapter(ureg) { + } + + class ThreadContext { + private: + QSBR::Context m_qsbrContext; + + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + m_qsbrContext = DefaultQSBR.createContext(); + } + + void unregisterThread() { + DefaultQSBR.destroyContext(m_qsbrContext); + } + + void update() { + DefaultQSBR.update(m_qsbrContext); + } + }; + + typedef ConcurrentMap_Linear Map; + + static ureg getInitialCapacity(ureg maxPopulation) { + return turf::util::roundUpPowerOf2(maxPopulation / 4); + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_H diff --git a/junction/extra/impl/MapAdapter_Linear_Mutex.h b/junction/extra/impl/MapAdapter_Linear_Mutex.h new file mode 100644 index 0000000..4339596 --- /dev/null +++ b/junction/extra/impl/MapAdapter_Linear_Mutex.h @@ -0,0 +1,79 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_MUTEX_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_MUTEX_H + +#include +#include +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Single + Mutex"; + + MapAdapter(ureg) { + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + } + + void unregisterThread() { + } + + void update() { + } + }; + + class Map { + private: + turf::Mutex m_mutex; + SingleMap_Linear m_map; + + public: + Map(ureg capacity) : m_map(capacity) { + } + + void insert(u32 key, void* value) { + turf::LockGuard guard(m_mutex); + m_map.insert(key, value); + } + + void* get(u32 key) { + turf::LockGuard guard(m_mutex); + return m_map.get(key); + } + + void* erase(u32 key) { + turf::LockGuard guard(m_mutex); + return m_map.erase(key); + } + }; + + static ureg getInitialCapacity(ureg maxPopulation) { + return turf::util::roundUpPowerOf2(maxPopulation / 4); + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_MUTEX_H diff --git a/junction/extra/impl/MapAdapter_Linear_RWLock.h b/junction/extra/impl/MapAdapter_Linear_RWLock.h new file mode 100644 index 0000000..b4e34aa --- /dev/null +++ b/junction/extra/impl/MapAdapter_Linear_RWLock.h @@ -0,0 +1,79 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_RWLOCK_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_RWLOCK_H + +#include +#include +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Single + RWLock"; + + MapAdapter(ureg) { + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + } + + void unregisterThread() { + } + + void update() { + } + }; + + class Map { + private: + turf::RWLock m_rwLock; + SingleMap_Linear m_map; + + public: + Map(ureg capacity) : m_map(capacity) { + } + + void insert(u32 key, void* value) { + turf::ExclusiveLockGuard guard(m_rwLock); + m_map.insert(key, value); + } + + void* get(u32 key) { + turf::SharedLockGuard guard(m_rwLock); + return m_map.get(key); + } + + void erase(u32 key) { + turf::ExclusiveLockGuard guard(m_rwLock); + m_map.erase(key); + } + }; + + static ureg getInitialCapacity(ureg maxPopulation) { + return turf::util::roundUpPowerOf2(maxPopulation / 4); + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_LINEAR_RWLOCK_H diff --git a/junction/extra/impl/MapAdapter_NBDS.h b/junction/extra/impl/MapAdapter_NBDS.h new file mode 100644 index 0000000..43ed97b --- /dev/null +++ b/junction/extra/impl/MapAdapter_NBDS.h @@ -0,0 +1,93 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_NBDS_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_NBDS_H + +#include + +#if !JUNCTION_WITH_NBDS +#error "You must configure with JUNCTION_WITH_NBDS=1!" +#endif + +extern "C" { +#include +#include +#include <../runtime/rlocal.h> +#include +} + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "nbds hashtable_t"; + + MapAdapter(ureg) { + } + + class ThreadContext { + private: + ureg m_threadIndex; + + public: + ThreadContext(MapAdapter&, ureg threadIndex) : m_threadIndex(threadIndex) { + } + + void registerThread() { + rcu_thread_init(m_threadIndex); + } + + void unregisterThread() { + } + + void update() { + rcu_update(); + } + }; + + class Map { + private: + hashtable_t* m_map; + + public: + Map(ureg) { + m_map = ht_alloc(NULL); + } + + ~Map() { + ht_free(m_map); + } + + void insert(u32 key, void* value) { + ht_cas(m_map, key, CAS_EXPECT_WHATEVER, (map_val_t) value); + } + + void* get(u32 key) { + return (void*) ht_get(m_map, key); + } + + void erase(u32 key) { + ht_remove(m_map, key); + } + }; + + static ureg getInitialCapacity(ureg) { + return 0; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_NBDS_H diff --git a/junction/extra/impl/MapAdapter_Null.h b/junction/extra/impl/MapAdapter_Null.h new file mode 100644 index 0000000..524f6e3 --- /dev/null +++ b/junction/extra/impl/MapAdapter_Null.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_NULL_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_NULL_H + +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Null"; + + MapAdapter(ureg) { + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + } + + void unregisterThread() { + } + + void update() { + } + }; + + class Map { + public: + Map(ureg) { + } + + void insert(u32, void*) { + } + + void* get(u32) { + return NULL; + } + + void erase(u32) { + } + }; + + static ureg getInitialCapacity(ureg) { + return 0; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_NULL_H diff --git a/junction/extra/impl/MapAdapter_StdMap.h b/junction/extra/impl/MapAdapter_StdMap.h new file mode 100644 index 0000000..4d287cc --- /dev/null +++ b/junction/extra/impl/MapAdapter_StdMap.h @@ -0,0 +1,106 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_STDMAP_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_STDMAP_H + +#include +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "std::map + std::mutex"; + + MapAdapter(ureg) { + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + } + + void unregisterThread() { + } + + void update() { + } + }; + + class Map { + private: + std::mutex m_mutex; + typedef std::map MapType; + MapType m_map; + + public: + Map(ureg) { + } + + void insert(u32 key, void* value) { + std::lock_guard guard(m_mutex); + m_map.insert(std::make_pair(key, value)); + } + + void* get(u32 key) { + std::lock_guard guard(m_mutex); + MapType::iterator iter = m_map.find(key); + return (iter == m_map.end()) ? NULL : iter->second; + } + + void erase(u32 key) { + std::lock_guard guard(m_mutex); + m_map.erase(key); + } + + class Iterator { + private: + Map& m_map; + MapType::iterator m_iter; + + public: + Iterator(Map& map) : m_map(map), m_iter(m_map.m_map.begin()) { + } + + void next() { + m_iter++; + } + + bool isValid() const { + return m_iter != m_map.m_map.end(); + } + + u32 getKey() const { + return m_iter->first; + } + + void* getValue() const { + return m_iter->second; + } + }; + }; + + static ureg getInitialCapacity(ureg) { + return 0; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_STDMAP_H diff --git a/junction/extra/impl/MapAdapter_TBB.h b/junction/extra/impl/MapAdapter_TBB.h new file mode 100644 index 0000000..a87a465 --- /dev/null +++ b/junction/extra/impl/MapAdapter_TBB.h @@ -0,0 +1,82 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_TBB_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_TBB_H + +#include + +#if !JUNCTION_WITH_TBB +#error "You must configure with JUNCTION_WITH_TBB=1!" +#endif + +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Intel TBB concurrent_hash_map"; + + MapAdapter(ureg) { + } + + class ThreadContext { + public: + ThreadContext(MapAdapter&, ureg) { + } + + void registerThread() { + } + + void unregisterThread() { + } + + void update() { + } + }; + + class Map { + private: + tbb::concurrent_hash_map m_map; + + public: + Map(ureg capacity) : m_map(capacity) { + } + + void insert(u32 key, void* value) { + m_map.insert(std::make_pair(key, value)); + } + + void* get(u32 key) { + tbb::concurrent_hash_map::const_accessor result; + if (m_map.find(result, key)) + return result->second; + else + return NULL; + } + + void erase(u32 key) { + m_map.erase(key); + } + }; + + static ureg getInitialCapacity(ureg maxPopulation) { + return maxPopulation / 4; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_TBB_H diff --git a/junction/extra/impl/MapAdapter_Tervel.h b/junction/extra/impl/MapAdapter_Tervel.h new file mode 100644 index 0000000..d00320d --- /dev/null +++ b/junction/extra/impl/MapAdapter_Tervel.h @@ -0,0 +1,90 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_EXTRA_IMPL_MAPADAPTER_TERVEL_H +#define JUNCTION_EXTRA_IMPL_MAPADAPTER_TERVEL_H + +#include + +#if !JUNCTION_WITH_TERVEL +#error "You must configure with JUNCTION_WITH_TERVEL=1!" +#endif + +#include +#include + +namespace junction { +namespace extra { + +class MapAdapter { +public: + static TURF_CONSTEXPR const char* MapName = "Tervel HashMap"; + + tervel::Tervel m_tervel; + + MapAdapter(ureg numThreads) : m_tervel(numThreads) { + } + + class ThreadContext { + public: + MapAdapter& m_adapter; + tervel::ThreadContext* m_context; + + ThreadContext(MapAdapter& adapter, ureg) : m_adapter(adapter), m_context(NULL) { + } + + void registerThread() { + m_context = new tervel::ThreadContext(&m_adapter.m_tervel); + } + + void unregisterThread() { + delete m_context; + } + + void update() { + } + }; + + class Map { + private: + tervel::containers::wf::HashMap m_map; + + public: + Map(ureg capacity) : m_map(capacity, 3) { + } + + void insert(u32 key, void* value) { + m_map.insert(key, (u64) value); + } + + void* get(u32 key) { + typename tervel::containers::wf::HashMap::ValueAccessor va; + if (m_map.at(key, va)) + return (void*) *va.value(); + else + return NULL; + } + + void erase(u32 key) { + m_map.remove(key); + } + }; + + static ureg getInitialCapacity(ureg maxPopulation) { + return maxPopulation / 4; + } +}; + +} // namespace extra +} // namespace junction + +#endif // JUNCTION_EXTRA_IMPL_MAPADAPTER_TERVEL_H diff --git a/junction/striped/AutoResetEvent.h b/junction/striped/AutoResetEvent.h new file mode 100644 index 0000000..ca7e0dd --- /dev/null +++ b/junction/striped/AutoResetEvent.h @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_STRIPED_AUTORESETEVENT_H +#define JUNCTION_STRIPED_AUTORESETEVENT_H + +#include +#include + +namespace junction { +namespace striped { + +class AutoResetEvent { +private: + JUNCTION_STRIPED_CONDITIONBANK_DEFINE_MEMBER() + bool m_status; + +public: + AutoResetEvent(bool status) : m_status(status) { + } + + void wait() { + ConditionPair& pair = JUNCTION_STRIPED_CONDITIONBANK_GET(this); + turf::LockGuard guard(pair.mutex); + while (!m_status) + pair.condVar.wait(guard); + m_status = false; + } + + void signal() { + ConditionPair& pair = JUNCTION_STRIPED_CONDITIONBANK_GET(this); + turf::LockGuard guard(pair.mutex); + if (!m_status) { + m_status = true; + // FIXME: Is there a more efficient striped::Mutex implementation that is safe from deadlock? + // This approach will wake up too many threads when there is heavy contention. + // However, if we don't wakeAll(), we will miss wakeups, since ConditionPairs are shared. + pair.condVar.wakeAll(); + } + } +}; + +} // namespace striped +} // namespace junction + +#endif // JUNCTION_STRIPED_AUTORESETEVENT_H diff --git a/junction/striped/ConditionBank.cpp b/junction/striped/ConditionBank.cpp new file mode 100644 index 0000000..30aec87 --- /dev/null +++ b/junction/striped/ConditionBank.cpp @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include + +#if JUNCTION_USE_STRIPING + +namespace junction { +namespace striped { + +ConditionBank DefaultConditionBank; + +ConditionPair* ConditionBank::initialize() { + m_initSpinLock.lock(); + ConditionPair* pairs = m_pairs.loadNonatomic(); + if (!pairs) { + pairs = new ConditionPair[SizeMask + 1]; + m_pairs.store(pairs, turf::Release); + } + m_initSpinLock.unlock(); + return pairs; +} + +} // namespace striped +} // namespace junction + +#endif // JUNCTION_USE_STRIPING diff --git a/junction/striped/ConditionBank.h b/junction/striped/ConditionBank.h new file mode 100644 index 0000000..1c3e6d3 --- /dev/null +++ b/junction/striped/ConditionBank.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_STRIPED_CONDITIONBANK_H +#define JUNCTION_STRIPED_CONDITIONBANK_H + +#include +#include + +#if JUNCTION_USE_STRIPING + +//----------------------------------- +// Striping enabled +//----------------------------------- +#include +#include + +namespace junction { +namespace striped { + +class ConditionBank { +private: + static const ureg SizeMask = 1023; + turf::Mutex_SpinLock m_initSpinLock; + turf::Atomic m_pairs; + + ConditionPair* initialize(); + +public: + ConditionPair& get(void* ptr) { + ConditionPair* pairs = m_pairs.load(turf::Consume); + if (!pairs) { + pairs = initialize(); + } + ureg index = turf::util::avalanche(uptr(ptr)) & SizeMask; + return pairs[index]; + } +}; + +extern ConditionBank DefaultConditionBank; + +} // namespace striped +} // namespace junction + +#define JUNCTION_STRIPED_CONDITIONBANK_DEFINE_MEMBER() +#define JUNCTION_STRIPED_CONDITIONBANK_GET(objectPtr) (junction::striped::DefaultConditionBank.get(objectPtr)) + +#else // JUNCTION_USE_STRIPING + +//----------------------------------- +// Striping disabled +//----------------------------------- +#define JUNCTION_STRIPED_CONDITIONBANK_DEFINE_MEMBER() junction::striped::ConditionPair m_conditionPair; +#define JUNCTION_STRIPED_CONDITIONBANK_GET(objectPtr) ((objectPtr)->m_conditionPair) + +#endif // JUNCTION_USE_STRIPING + +#endif // JUNCTION_STRIPED_CONDITIONBANK_H diff --git a/junction/striped/ConditionPair.h b/junction/striped/ConditionPair.h new file mode 100644 index 0000000..c453cd7 --- /dev/null +++ b/junction/striped/ConditionPair.h @@ -0,0 +1,31 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_STRIPED_CONDITIONPAIR_H +#define JUNCTION_STRIPED_CONDITIONPAIR_H + +#include +#include +#include + +namespace junction { +namespace striped { + +struct ConditionPair { + turf::Mutex mutex; + turf::ConditionVariable condVar; +}; + +} // namespace striped +} // namespace junction + +#endif // JUNCTION_STRIPED_CONDITIONPAIR_H diff --git a/junction/striped/ManualResetEvent.h b/junction/striped/ManualResetEvent.h new file mode 100644 index 0000000..e902f9a --- /dev/null +++ b/junction/striped/ManualResetEvent.h @@ -0,0 +1,99 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_STRIPED_MANUALRESETEVENT_H +#define JUNCTION_STRIPED_MANUALRESETEVENT_H + +#include + +#if JUNCTION_USE_STRIPING + +//----------------------------------- +// Striping enabled +//----------------------------------- +#include + +namespace junction { +namespace striped { + +class ManualResetEvent { +private: + JUNCTION_STRIPED_CONDITIONBANK_DEFINE_MEMBER() + static const u8 Signaled = 1; + static const u8 HasWaiters = 2; + turf::Atomic m_state; + +public: + ManualResetEvent(bool initialState = false) : m_state(initialState ? Signaled : 0) { + } + + ~ManualResetEvent() { + } + + void signal() { + u8 prevState = m_state.fetchOr(Signaled, turf::Release); // Synchronizes-with the load in wait (fast path) + if (prevState & HasWaiters) { + ConditionPair& pair = JUNCTION_STRIPED_CONDITIONBANK_GET(this); + turf::LockGuard guard(pair.mutex); // Prevents the wake from occuring in the middle of wait()'s critical section + pair.condVar.wakeAll(); + } + } + + bool isSignaled() const { + return m_state.load(turf::Relaxed) & Signaled; + } + + void reset() { + TURF_ASSERT(0); // FIXME: implement it + } + + void wait() { + u8 state = m_state.load(turf::Acquire); // Synchronizes-with the fetchOr in signal (fast path) + if ((state & Signaled) == 0) { + ConditionPair& pair = JUNCTION_STRIPED_CONDITIONBANK_GET(this); + turf::LockGuard guard(pair.mutex); + for (;;) { + // FIXME: Implement reusable AdaptiveBackoff class and apply it here + state = m_state.load(turf::Relaxed); + if (state & Signaled) + break; + if (state != HasWaiters) { + TURF_ASSERT(state == 0); + if (!m_state.compareExchangeWeak(state, HasWaiters, turf::Relaxed, turf::Relaxed)) + continue; + } + // The lock ensures signal can't wakeAll between the load and the wait + pair.condVar.wait(guard); + } + } + } +}; + +} // namespace striped +} // namespace junction + +#else // JUNCTION_USE_STRIPING + +//----------------------------------- +// Striping disabled +//----------------------------------- +#include + +namespace junction { +namespace striped { +typedef turf::ManualResetEvent ManualResetEvent; +} // namespace striped +} // namespace junction + +#endif // JUNCTION_USE_STRIPING + +#endif // JUNCTION_STRIPED_MANUALRESETEVENT_H diff --git a/junction/striped/Mutex.h b/junction/striped/Mutex.h new file mode 100644 index 0000000..c88b1ab --- /dev/null +++ b/junction/striped/Mutex.h @@ -0,0 +1,76 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef JUNCTION_STRIPED_MUTEX_H +#define JUNCTION_STRIPED_MUTEX_H + +#include + +#if JUNCTION_USE_STRIPING + +//----------------------------------- +// Striping enabled +//----------------------------------- +#include + +namespace junction { +namespace striped { + +// Not recursive +class Mutex { +private: + turf::Atomic m_status; + junction::striped::AutoResetEvent m_event; + + void lockSlow() { + while (m_status.exchange(1, turf::Acquire) >= 0) + m_event.wait(); + } + +public: + Mutex() : m_status(-1), m_event(false) { + } + + void lock() { + if (m_status.exchange(0, turf::Acquire) >= 0) + lockSlow(); + } + + bool tryLock() { + return (m_status.compareExchange(-1, 0, turf::Acquire) < 0); + } + + void unlock() { + if (m_status.exchange(-1, turf::Release) > 0) + m_event.signal(); + } +}; + +} // namespace striped +} // namespace junction + +#else // JUNCTION_USE_STRIPING + +//----------------------------------- +// Striping disabled +//----------------------------------- +#include + +namespace junction { +namespace striped { +typedef turf::Mutex Mutex; +} // namespace striped +} // namespace junction + +#endif // JUNCTION_USE_STRIPING + +#endif // JUNCTION_STRIPED_MUTEX_H diff --git a/samples/MallocTest/CMakeLists.txt b/samples/MallocTest/CMakeLists.txt new file mode 100644 index 0000000..9838328 --- /dev/null +++ b/samples/MallocTest/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8.5) + +get_filename_component(SAMPLE_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # CMAKE_CONFIGURATION_TYPES only reliable if set before project(), and not from an include file + set(CMAKE_CONFIGURATION_TYPES "Debug;RelWithAsserts;RelWithDebInfo" CACHE INTERNAL "Build configs") + project(${SAMPLE_NAME}) +endif() + +include(../../cmake/AddSample.cmake) diff --git a/samples/MallocTest/MallocTest.cpp b/samples/MallocTest/MallocTest.cpp new file mode 100644 index 0000000..37efc77 --- /dev/null +++ b/samples/MallocTest/MallocTest.cpp @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include +#include + +using namespace turf::intTypes; + +int main() { + junction::extra::MapAdapter adapter(1); + junction::extra::MapAdapter::ThreadContext context(adapter, 0); + junction::extra::MapAdapter::Map map(65536); + + ureg population = 0; + for (ureg i = 0; i < 100; i++) { +#if TURF_USE_DLMALLOC && TURF_DLMALLOC_FAST_STATS + std::cout << "Population=" << population << ", inUse=" << TURF_HEAP.getInUseBytes() << std::endl; +#endif + for (; population < i * 5000; population++) + map.insert(population + 1, (void*) ((population << 2) | 3)); + } + + return 0; +} diff --git a/samples/MapCorrectnessTests/CMakeLists.txt b/samples/MapCorrectnessTests/CMakeLists.txt new file mode 100644 index 0000000..b1c771c --- /dev/null +++ b/samples/MapCorrectnessTests/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 2.8.5) + +get_filename_component(SAMPLE_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # CMAKE_CONFIGURATION_TYPES only reliable if set before project(), and not from an include file + set(CMAKE_CONFIGURATION_TYPES "Debug;RelWithAsserts;RelWithDebInfo" CACHE INTERNAL "Build configs") + project(${SAMPLE_NAME}) +endif() + +set(TEST_CHECK_MAP_CONTENTS TRUE CACHE BOOL "Validate contents of the map using its iterator") +set(JUNCTION_USERCONFIG "junction_userconfig.h.in" CACHE STRING "Custom config for ${SAMPLE_NAME}") +include(../../cmake/AddSample.cmake) diff --git a/samples/MapCorrectnessTests/MapCorrectnessTests.cpp b/samples/MapCorrectnessTests/MapCorrectnessTests.cpp new file mode 100644 index 0000000..ca7f9d2 --- /dev/null +++ b/samples/MapCorrectnessTests/MapCorrectnessTests.cpp @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include "TestEnvironment.h" +#include "TestInsertSameKeys.h" +#include "TestInsertDifferentKeys.h" +#include "TestChurn.h" +#include +#include // for GrampaStats + +static const ureg IterationsPerLog = 100; + +int main(int argc, const char** argv) { + TestEnvironment env; + + TestInsertSameKeys testInsertSameKeys(env); + TestInsertDifferentKeys testInsertDifferentKeys(env); + TestChurn testChurn(env); + for (;;) { + for (ureg c = 0; c < IterationsPerLog; c++) { + testInsertSameKeys.run(); + testInsertDifferentKeys.run(); + testChurn.run(); + } + turf::Trace::Instance.dumpStats(); + +#if JUNCTION_TRACK_GRAMPA_STATS + junction::DefaultQSBR.flush(); + junction::details::GrampaStats& stats = junction::details::GrampaStats::Instance; + printf("---------------------------\n"); + printf("numTables: %d/%d\n", (int) stats.numTables.current.load(turf::Relaxed), (int) stats.numTables.total.load(turf::Relaxed)); + printf("numTableMigrations: %d/%d\n", (int) stats.numTableMigrations.current.load(turf::Relaxed), (int) stats.numTableMigrations.total.load(turf::Relaxed)); + printf("numFlatTrees: %d/%d\n", (int) stats.numFlatTrees.current.load(turf::Relaxed), (int) stats.numFlatTrees.total.load(turf::Relaxed)); + printf("numFlatTreeMigrations: %d/%d\n", (int) stats.numFlatTreeMigrations.current.load(turf::Relaxed), (int) stats.numFlatTreeMigrations.total.load(turf::Relaxed)); +#endif + } + + return 0; +} diff --git a/samples/MapCorrectnessTests/TestChurn.h b/samples/MapCorrectnessTests/TestChurn.h new file mode 100644 index 0000000..5b93ff4 --- /dev/null +++ b/samples/MapCorrectnessTests/TestChurn.h @@ -0,0 +1,173 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef SAMPLES_MAPCORRECTNESSTESTS_TESTCHURN_H +#define SAMPLES_MAPCORRECTNESSTESTS_TESTCHURN_H + +#include +#include "TestEnvironment.h" +#include +#include +#include + +class TestChurn { +public: + static const ureg KeysInBlock = 32; + static const ureg BlocksToMaintain = 256; + static const ureg BlocksToLookup = 4; + static const ureg StepsPerIteration = 100; + + enum Phase { + Phase_Insert, + Phase_Lookup, + Phase_Erase, + Phase_LookupDeleted, + }; + + struct ThreadInfo { + turf::extra::Random random; + u32 rangeLo; + u32 rangeHi; + u32 insertIndex; + u32 eraseIndex; + u32 lookupIndex; + Phase phase; + ureg keysToCheck; + }; + + TestEnvironment& m_env; + MapAdapter::Map m_map; + u32 m_rangePerThread; + u32 m_relativePrime; + std::vector m_threads; + + TestChurn(TestEnvironment& env) : m_env(env), m_map(MapAdapter::getInitialCapacity(KeysInBlock * BlocksToMaintain * env.numThreads)) { + m_threads.resize(m_env.numThreads); + m_rangePerThread = u32(-3) / m_env.numThreads; // from 2 to 0xffffffff inclusive + TURF_ASSERT(KeysInBlock * (BlocksToMaintain + BlocksToLookup + 1) < m_rangePerThread); + u32 startIndex = 2; + for (ureg i = 0; i < m_env.numThreads; i++) { + ThreadInfo& thread = m_threads[i]; + thread.rangeLo = startIndex; + startIndex += m_rangePerThread; + thread.rangeHi = startIndex; + thread.insertIndex = thread.rangeLo + thread.random.next32() % m_rangePerThread; + thread.eraseIndex = thread.insertIndex; + thread.lookupIndex = 0; + thread.phase = Phase_Insert; + thread.keysToCheck = 0; + } + m_relativePrime = m_threads[0].random.next32() * 2 + 1; + m_env.dispatcher.kick(&TestChurn::warmUp, *this); + } + + void warmUp(ureg threadIndex) { + ThreadInfo& thread = m_threads[threadIndex]; + TURF_ASSERT(thread.phase == Phase_Insert); + TURF_ASSERT(thread.insertIndex == thread.eraseIndex); + for (sreg keysRemaining = KeysInBlock * BlocksToMaintain; keysRemaining > 0; keysRemaining--) { + u32 key = thread.insertIndex * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { + m_map.insert(key, (void*) uptr(key)); + } + if (++thread.insertIndex >= thread.rangeHi) + thread.insertIndex = thread.rangeLo; + } + } + + void doChurn(ureg threadIndex) { + ThreadInfo& thread = m_threads[threadIndex]; + TURF_ASSERT(thread.insertIndex != thread.eraseIndex); + for (sreg stepsRemaining = StepsPerIteration; stepsRemaining > 0; stepsRemaining--) { + switch (thread.phase) { + case Phase_Insert: { + for (sreg keysRemaining = KeysInBlock; keysRemaining > 0; keysRemaining--) { + u32 key = thread.insertIndex * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { + m_map.insert(key, (void*) uptr(key)); + } + if (++thread.insertIndex >= thread.rangeHi) + thread.insertIndex = thread.rangeLo; + TURF_ASSERT(thread.insertIndex != thread.eraseIndex); + } + thread.phase = Phase_Lookup; + thread.lookupIndex = thread.insertIndex; + thread.keysToCheck = KeysInBlock + (thread.random.next32() % (KeysInBlock * (BlocksToLookup - 1))); + break; + } + case Phase_Lookup: { + sreg keysRemaining = turf::util::min(thread.keysToCheck, KeysInBlock); + thread.keysToCheck -= keysRemaining; + for (; keysRemaining > 0; keysRemaining--) { + if (thread.lookupIndex == thread.rangeLo) + thread.lookupIndex = thread.rangeHi; + thread.lookupIndex--; + u32 key = thread.lookupIndex * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { + if (m_map.get(key) != (void*) uptr(key)) + TURF_DEBUG_BREAK(); + } + } + if (thread.keysToCheck == 0) { + thread.phase = Phase_Erase; + } + break; + } + case Phase_Erase: { + for (sreg keysRemaining = KeysInBlock; keysRemaining > 0; keysRemaining--) { + u32 key = thread.eraseIndex * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { + m_map.erase(key); + } + if (++thread.eraseIndex >= thread.rangeHi) + thread.eraseIndex = thread.rangeLo; + TURF_ASSERT(thread.insertIndex != thread.eraseIndex); + } + thread.phase = Phase_LookupDeleted; + thread.lookupIndex = thread.eraseIndex; + thread.keysToCheck = KeysInBlock + (thread.random.next32() % (KeysInBlock * (BlocksToLookup - 1))); + break; + } + case Phase_LookupDeleted: { + sreg keysRemaining = turf::util::min(thread.keysToCheck, KeysInBlock); + thread.keysToCheck -= keysRemaining; + for (; keysRemaining > 0; keysRemaining--) { + if (thread.lookupIndex == thread.rangeLo) + thread.lookupIndex = thread.rangeHi; + thread.lookupIndex--; + u32 key = thread.lookupIndex * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { + if (m_map.get(key)) + TURF_DEBUG_BREAK(); + } + } + if (thread.keysToCheck == 0) { + thread.phase = Phase_Insert; + } + break; + } + } + } + m_env.threads[threadIndex].update(); + } + + void run() { + m_env.dispatcher.kick(&TestChurn::doChurn, *this); + } +}; + +#endif // SAMPLES_MAPCORRECTNESSTESTS_TESTCHURN_H diff --git a/samples/MapCorrectnessTests/TestEnvironment.h b/samples/MapCorrectnessTests/TestEnvironment.h new file mode 100644 index 0000000..6246272 --- /dev/null +++ b/samples/MapCorrectnessTests/TestEnvironment.h @@ -0,0 +1,43 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef SAMPLES_MAPCORRECTNESSTESTS_TESTENVIRONMENT_H +#define SAMPLES_MAPCORRECTNESSTESTS_TESTENVIRONMENT_H + +#include +#include +#include +#include + +using namespace turf::intTypes; +typedef junction::extra::MapAdapter MapAdapter; + +struct TestEnvironment { + turf::extra::JobDispatcher dispatcher; + ureg numThreads; + MapAdapter adapter; + std::vector threads; + + TestEnvironment() : numThreads(dispatcher.getNumPhysicalCores()), adapter(numThreads) { + TURF_ASSERT(numThreads > 0); + for (ureg t = 0; t < numThreads; t++) + threads.emplace_back(adapter, t); + dispatcher.kickMulti(&MapAdapter::ThreadContext::registerThread, &threads[0], threads.size()); + } + + ~TestEnvironment() { + dispatcher.kickMulti(&MapAdapter::ThreadContext::unregisterThread, &threads[0], threads.size()); + } +}; + + +#endif // SAMPLES_MAPCORRECTNESSTESTS_TESTENVIRONMENT_H diff --git a/samples/MapCorrectnessTests/TestInsertDifferentKeys.h b/samples/MapCorrectnessTests/TestInsertDifferentKeys.h new file mode 100644 index 0000000..df21e29 --- /dev/null +++ b/samples/MapCorrectnessTests/TestInsertDifferentKeys.h @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef SAMPLES_MAPCORRECTNESSTESTS_TESTINSERTDIFFERENTKEYS_H +#define SAMPLES_MAPCORRECTNESSTESTS_TESTINSERTDIFFERENTKEYS_H + +#include +#include "TestEnvironment.h" +#include + +class TestInsertDifferentKeys { +public: + static const ureg KeysToInsert = 2048; + TestEnvironment& m_env; + MapAdapter::Map *m_map; + turf::extra::Random m_random; + u32 m_startIndex; + u32 m_relativePrime; + + TestInsertDifferentKeys(TestEnvironment& env) : m_env(env), m_map(NULL), m_startIndex(0), m_relativePrime(0) { + } + + void insertKeys(ureg threadIndex) { + u32 index = m_startIndex + threadIndex * (KeysToInsert + 2); + sreg keysRemaining = KeysToInsert; + while (keysRemaining > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + m_map->insert(key, (void*) uptr(key)); + keysRemaining--; + } + index++; + } + m_env.threads[threadIndex].update(); + } + + void eraseKeys(ureg threadIndex) { + u32 index = m_startIndex + threadIndex * (KeysToInsert + 2); + sreg keysRemaining = KeysToInsert; + while (keysRemaining > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + m_map->erase(key); + keysRemaining--; + } + index++; + } + m_env.threads[threadIndex].update(); + } + + void checkMapContents() { +#if TEST_CHECK_MAP_CONTENTS + ureg iterCount = 0; + ureg iterChecksum = 0; + for (MapAdapter::Map::Iterator iter(*m_map); iter.isValid(); iter.next()) { + iterCount++; + u32 key = iter.getKey(); + iterChecksum += key; + if (iter.getValue() != (void*) uptr(key)) + TURF_DEBUG_BREAK(); + } + + ureg actualChecksum = 0; + for (ureg i = 0; i < m_env.numThreads; i++) { + u32 index = m_startIndex + i * (KeysToInsert + 2); + sreg leftToCheck = KeysToInsert; + while (leftToCheck > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + if (m_map->get(key) != (void*) uptr(key)) + TURF_DEBUG_BREAK(); + actualChecksum += key; + leftToCheck--; + } + index++; + } + } + + if (iterCount != KeysToInsert * m_env.numThreads) + TURF_DEBUG_BREAK(); + if (iterChecksum != actualChecksum) + TURF_DEBUG_BREAK(); +#endif + } + + void checkMapEmpty() { +#if TEST_CHECK_MAP_CONTENTS + for (MapAdapter::Map::Iterator iter(*m_map); iter.isValid(); iter.next()) { + TURF_DEBUG_BREAK(); + } + + for (ureg i = 0; i < m_env.numThreads; i++) { + u32 index = m_startIndex + i * (KeysToInsert + 2); + sreg leftToCheck = KeysToInsert; + while (leftToCheck > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + if (m_map->get(key)) + TURF_DEBUG_BREAK(); + leftToCheck--; + } + index++; + } + } +#endif // TEST_CHECK_MAP_CONTENTS + } + + void run() { + m_map = new MapAdapter::Map(MapAdapter::getInitialCapacity(KeysToInsert)); + m_startIndex = m_random.next32(); + m_relativePrime = m_random.next32() * 2 + 1; + m_env.dispatcher.kick(&TestInsertDifferentKeys::insertKeys, *this); + checkMapContents(); + m_env.dispatcher.kick(&TestInsertDifferentKeys::eraseKeys, *this); + checkMapEmpty(); + delete m_map; + m_map = NULL; + } +}; + +#endif // SAMPLES_MAPCORRECTNESSTESTS_TESTINSERTDIFFERENTKEYS_H diff --git a/samples/MapCorrectnessTests/TestInsertSameKeys.h b/samples/MapCorrectnessTests/TestInsertSameKeys.h new file mode 100644 index 0000000..3e696af --- /dev/null +++ b/samples/MapCorrectnessTests/TestInsertSameKeys.h @@ -0,0 +1,130 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#ifndef SAMPLES_MAPCORRECTNESSTESTS_TESTINSERTSAMEKEYS_H +#define SAMPLES_MAPCORRECTNESSTESTS_TESTINSERTSAMEKEYS_H + +#include +#include "TestEnvironment.h" +#include + +class TestInsertSameKeys { +public: + static const ureg KeysToInsert = 2048; + TestEnvironment& m_env; + MapAdapter::Map *m_map; + turf::extra::Random m_random; + u32 m_startIndex; + u32 m_relativePrime; + + TestInsertSameKeys(TestEnvironment& env) : m_env(env), m_map(NULL), m_startIndex(0), m_relativePrime(0) { + } + + void insertKeys(ureg threadIndex) { + u32 index = m_startIndex; + sreg keysRemaining = KeysToInsert; + while (keysRemaining > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + m_map->insert(key, (void*) uptr(key)); + keysRemaining--; + } + index++; + } + m_env.threads[threadIndex].update(); + } + + void eraseKeys(ureg threadIndex) { + u32 index = m_startIndex; + sreg keysRemaining = KeysToInsert; + while (keysRemaining > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + m_map->erase(key); + keysRemaining--; + } + index++; + } + m_env.threads[threadIndex].update(); + } + + void checkMapContents() { +#if TEST_CHECK_MAP_CONTENTS + ureg iterCount = 0; + ureg iterChecksum = 0; + for (MapAdapter::Map::Iterator iter(*m_map); iter.isValid(); iter.next()) { + iterCount++; + u32 key = iter.getKey(); + iterChecksum += key; + if (iter.getValue() != (void*) uptr(key)) + TURF_DEBUG_BREAK(); + } + + ureg actualChecksum = 0; + u32 index = m_startIndex; + sreg leftToCheck = KeysToInsert; + while (leftToCheck > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + if (m_map->get(key) != (void*) uptr(key)) + TURF_DEBUG_BREAK(); + actualChecksum += key; + leftToCheck--; + } + index++; + } + + if (iterCount != KeysToInsert) + TURF_DEBUG_BREAK(); + if (iterChecksum != actualChecksum) + TURF_DEBUG_BREAK(); +#endif // TEST_CHECK_MAP_CONTENTS + } + + void checkMapEmpty() { +#if TEST_CHECK_MAP_CONTENTS + for (MapAdapter::Map::Iterator iter(*m_map); iter.isValid(); iter.next()) { + TURF_DEBUG_BREAK(); + } + + u32 index = m_startIndex; + sreg leftToCheck = KeysToInsert; + while (leftToCheck > 0) { + u32 key = index * m_relativePrime; + key = key ^ (key >> 16); + if (key >= 2) { // Don't insert 0 or 1 + if (m_map->get(key)) + TURF_DEBUG_BREAK(); + leftToCheck--; + } + index++; + } +#endif // TEST_CHECK_MAP_CONTENTS + } + + void run() { + m_map = new MapAdapter::Map(MapAdapter::getInitialCapacity(KeysToInsert)); + m_startIndex = m_random.next32(); + m_relativePrime = m_random.next32() * 2 + 1; + m_env.dispatcher.kick(&TestInsertSameKeys::insertKeys, *this); + checkMapContents(); + m_env.dispatcher.kick(&TestInsertSameKeys::eraseKeys, *this); + checkMapEmpty(); + delete m_map; + m_map = NULL; + } +}; + +#endif // SAMPLES_MAPCORRECTNESSTESTS_TESTINSERTSAMEKEYS_H diff --git a/samples/MapCorrectnessTests/junction_userconfig.h.in b/samples/MapCorrectnessTests/junction_userconfig.h.in new file mode 100644 index 0000000..3994dab --- /dev/null +++ b/samples/MapCorrectnessTests/junction_userconfig.h.in @@ -0,0 +1,4 @@ +#cmakedefine01 TEST_CHECK_MAP_CONTENTS + +#define TURF_IMPL_TRACE_PATH "impl/Trace_Counters.h" +#define TURF_IMPL_TRACE_TYPE turf::Trace_Counters diff --git a/samples/MapMemoryBench/CMakeLists.txt b/samples/MapMemoryBench/CMakeLists.txt new file mode 100644 index 0000000..9838328 --- /dev/null +++ b/samples/MapMemoryBench/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8.5) + +get_filename_component(SAMPLE_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # CMAKE_CONFIGURATION_TYPES only reliable if set before project(), and not from an include file + set(CMAKE_CONFIGURATION_TYPES "Debug;RelWithAsserts;RelWithDebInfo" CACHE INTERNAL "Build configs") + project(${SAMPLE_NAME}) +endif() + +include(../../cmake/AddSample.cmake) diff --git a/samples/MapMemoryBench/MapMemoryBench.cpp b/samples/MapMemoryBench/MapMemoryBench.cpp new file mode 100644 index 0000000..e0c8fdc --- /dev/null +++ b/samples/MapMemoryBench/MapMemoryBench.cpp @@ -0,0 +1,71 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include +#include +#include + +#if TURF_USE_DLMALLOC && TURF_DLMALLOC_FAST_STATS + +using namespace turf::intTypes; +typedef junction::extra::MapAdapter MapAdapter; + +static const ureg MaxPopulation = 1000000; +static const ureg MinPopulation = 1000; +static const ureg StepSize = 500; +static const u32 Prime = 0x4190ab09; + +int main() { + MapAdapter adapter(1); + MapAdapter::ThreadContext threadCtx(adapter, 0); + threadCtx.registerThread(); + + ureg startMem = TURF_HEAP.getInUseBytes(); + ureg mem = 0; + ureg initialCapacity = MapAdapter::getInitialCapacity(MaxPopulation); + MapAdapter::Map* map = new MapAdapter::Map(initialCapacity); + + printf("[\n"); + ureg population = 0; + while (population < MaxPopulation) { + ureg loMem = mem; + ureg hiMem = mem; + for (ureg target = population + StepSize; population < target; population++) { + u32 key = u32(population + 1) * Prime; + if (key >= 2) { + map->insert(key, (void*) uptr(key)); + population++; + threadCtx.update(); + ureg inUseBytes = TURF_HEAP.getInUseBytes(); + mem = inUseBytes - startMem; + loMem = turf::util::min(loMem, mem); + hiMem = turf::util::max(hiMem, mem); + } + } + printf(" (%d, %d, %d),\n", (int) population, (int) loMem, (int) hiMem); + } + printf("]\n"); + + delete map; + return 0; +} + +#else // TURF_USE_DLMALLOC && TURF_DLMALLOC_FAST_STATS + +int main() { + fprintf(stderr, "Must configure with TURF_USE_DLMALLOC=1 and TURF_DLMALLOC_FAST_STATS=1\n"); + return 1; +} + +#endif // TURF_USE_DLMALLOC && TURF_DLMALLOC_FAST_STATS diff --git a/samples/MapMemoryBench/RenderGraphs.py b/samples/MapMemoryBench/RenderGraphs.py new file mode 100644 index 0000000..03b05fe --- /dev/null +++ b/samples/MapMemoryBench/RenderGraphs.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +import os +import cairo +import math +import glob + + +#--------------------------------------------------- +# Cairo drawing helpers +#--------------------------------------------------- +def createScaledFont(family, size, slant=cairo.FONT_SLANT_NORMAL, weight=cairo.FONT_WEIGHT_NORMAL): + """ Simple helper function to create a cairo ScaledFont. """ + face = cairo.ToyFontFace(family, slant, weight) + DEFAULT_FONT_OPTIONS = cairo.FontOptions() + DEFAULT_FONT_OPTIONS.set_antialias(cairo.ANTIALIAS_SUBPIXEL) + return cairo.ScaledFont(face, cairo.Matrix(xx=size, yy=size), cairo.Matrix(), DEFAULT_FONT_OPTIONS) + +def fillAlignedText(cr, x, y, scaledFont, text, alignment = 0): + """ Draw some aligned text at the specified co-ordinates. + alignment = 0: left-justify + alignment = 0.5: center + alignment = 1: right-justify """ + ascent, descent = scaledFont.extents()[:2] + x_bearing, y_bearing, width, height = scaledFont.text_extents(text)[:4] + with Saved(cr): + cr.set_scaled_font(scaledFont) + cr.move_to(math.floor(x + 0.5 - width * alignment), math.floor(y + 0.5)) + cr.text_path(text) + cr.fill() + +class Saved(): + """ Preserve cairo state inside the scope of a with statement. """ + def __init__(self, cr): + self.cr = cr + def __enter__(self): + self.cr.save() + return self.cr + def __exit__(self, type, value, traceback): + self.cr.restore() + + +#--------------------------------------------------- +# AxisAttribs +#--------------------------------------------------- +class AxisAttribs: + """ Describes one axis on the graph. Can be linear or logarithmic. """ + + def __init__(self, size, min, max, step, logarithmic = False, labeler = lambda x: str(int(x + 0.5))): + self.size = float(size) + self.logarithmic = logarithmic + self.labeler = labeler + self.toAxis = lambda x: math.log(x) if logarithmic else float(x) + self.fromAxis = lambda x: math.exp(x) if logarithmic else float(x) + self.min = self.toAxis(min) + self.max = self.toAxis(max) + self.step = self.toAxis(step) + + def mapAxisValue(self, x): + """ Maps x to a point along the axis. + x should already have been filtered through self.toAxis(), especially if logarithmic. """ + return (x - self.min) / (self.max - self.min) * self.size + + def iterLabels(self): + """ Helper to iterate through all the tick marks along the axis. """ + lo = int(math.floor(self.min / self.step + 1 - 1e-9)) + hi = int(math.floor(self.max / self.step + 1e-9)) + for i in xrange(lo, hi + 1): + value = i * self.step + if self.min == 0 and i == 0: + continue + yield self.mapAxisValue(value), self.labeler(self.fromAxis(value)) + + +#--------------------------------------------------- +# Graph +#--------------------------------------------------- +class Curve: + def __init__(self, name, points, color): + self.name = name + self.points = points + self.color = color + +class Graph: + """ Renders a graph. """ + + def __init__(self, xAttribs, yAttribs): + self.xAttribs = xAttribs + self.yAttribs = yAttribs + self.curves = [] + + def addCurve(self, curve): + self.curves.append(curve) + + def renderTo(self, fileName): + xAttribs = self.xAttribs + yAttribs = self.yAttribs + + # Create the image surface and cairo context + surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, 140 + int(xAttribs.size + 0.5), 65 + int(yAttribs.size + 0.5)) + cr = cairo.Context(surface) + cr.set_source_rgb(1, 1, 1) + cr.paint() + cr.set_miter_limit(1.414) + cr.translate(58, 11 + yAttribs.size) + + # Draw axes + labelFont = createScaledFont('Arial', 11) + with Saved(cr): + cr.set_line_width(1) + cr.set_source_rgb(.4, .4, .4) + + # Horizontal axis + cr.move_to(0, -0.5) + cr.rel_line_to(xAttribs.size + 1, 0) + for pos, label in xAttribs.iterLabels(): # Tick marks + x = math.floor(pos + 0.5) + 0.5 + cr.move_to(x, -1) + cr.rel_line_to(0, 4) + cr.stroke() + for pos, label in xAttribs.iterLabels(): # Labels + x = math.floor(pos + 0.5) + with Saved(cr): + cr.translate(x - 1, 5) + cr.rotate(-math.pi / 4) + fillAlignedText(cr, 0, 6, labelFont, label, 1) + + # Vertical axis + cr.move_to(0.5, 0) + cr.rel_line_to(0, -yAttribs.size - 0.5) + for pos, label in yAttribs.iterLabels(): # Tick marks + if label == '0': + continue + y = -math.floor(pos + 0.5) - 0.5 + cr.move_to(1, y) + cr.rel_line_to(-4, 0) + cr.stroke() + for pos, label in yAttribs.iterLabels(): # Labels + if label == '0': + continue + fillAlignedText(cr, -4, -pos + 4, labelFont, label, 1) + + # Draw curves + for curve in self.curves: + points = curve.points + width = 2.5 + color = curve.color + with Saved(cr): + cr.set_line_width(width) + cr.set_source_rgba(*color) + with Saved(cr): + cr.rectangle(0, 5, xAttribs.size, -yAttribs.size - 15) + cr.clip() + cr.move_to(xAttribs.mapAxisValue(points[0][0]), -yAttribs.mapAxisValue(points[0][1])) + for x, y, yHi in points[1:]: + cr.line_to(xAttribs.mapAxisValue(x) + 0.5, -yAttribs.mapAxisValue(y) - 0.5) + cr.stroke() + + # Label + labelFont = createScaledFont('Arial', 11) + label = curve.name + x, y, yHi = points[-1] + fillAlignedText(cr, xAttribs.mapAxisValue(x) + 3, -yAttribs.mapAxisValue(y) + 4, labelFont, label, 0) + + # Draw axis names + cr.set_source_rgb(0, 0, 0) + axisFont = createScaledFont('Helvetica', 14, weight=cairo.FONT_WEIGHT_BOLD) + with Saved(cr): + cr.translate(-47, -yAttribs.size / 2.0) + cr.rotate(-math.pi / 2) + fillAlignedText(cr, 0, 0, axisFont, "Bytes In Use", 0.5) + fillAlignedText(cr, xAttribs.size / 2.0, 50, axisFont, "Population", 0.5) + + # Save PNG file + surface.write_to_png(fileName) + + +#--------------------------------------------------- +# main +#--------------------------------------------------- +graph = Graph(AxisAttribs(600, 0, 1000000, 200000), AxisAttribs(320, 0, 50000000, 10000000)) +COLORS = [ + (1, 0, 0), + (1, 0.5, 0), + (0.5, 0.5, 0), + (0, 1, 0), + (0, 0.5, 1), + (0, 0, 1), + (1, 0, 1) +] +for i, fn in enumerate(glob.glob('build*/results.txt')): + points = eval(open(fn, 'r').read()) + graph.addCurve(Curve(os.path.split(fn)[0], points, COLORS[i % len(COLORS)])) +graph.renderTo('out.png') diff --git a/samples/MapMemoryBench/TestAllMaps.py b/samples/MapMemoryBench/TestAllMaps.py new file mode 100644 index 0000000..1dfc5fa --- /dev/null +++ b/samples/MapMemoryBench/TestAllMaps.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +import os +import subprocess +import sys + +# Default CMake command is just 'cmake' but you can override it by setting +# the CMAKE environment variable: +CMAKE = os.getenv('CMAKE', 'cmake') + +ALL_MAPS = [ + ('michael', 'junction/extra/impl/MapAdapter_CDS_Michael.h', ['-DJUNCTION_WITH_CDS=1', '-DTURF_WITH_EXCEPTIONS=1']), + ('linear', 'junction/extra/impl/MapAdapter_Linear.h', []), + ('leapfrog', 'junction/extra/impl/MapAdapter_LeapFrog.h', []), + ('grampa', 'junction/extra/impl/MapAdapter_Grampa.h', []), + ('stdmap', 'junction/extra/impl/MapAdapter_StdMap.h', []), + ('folly', 'junction/extra/impl/MapAdapter_Folly.h', ['-DJUNCTION_WITH_FOLLY=1', '-DTURF_WITH_EXCEPTIONS=1']), + ('nbds', 'junction/extra/impl/MapAdapter_NBDS.h', ['-DJUNCTION_WITH_NBDS=1']), + ('tbb', 'junction/extra/impl/MapAdapter_TBB.h', ['-DJUNCTION_WITH_TBB=1']), + ('tervel', 'junction/extra/impl/MapAdapter_Tervel.h', ['-DJUNCTION_WITH_TERVEL=1']), +] + +# Scan arguments for path to CMakeLists.txt and args to pass through. +passThroughArgs = [] +pathArgs = [] +for arg in sys.argv[1:]: + if arg.startswith('-'): + passThroughArgs.append(arg) + else: + pathArgs.append(arg) +if len(pathArgs) != 1: + sys.stderr.write('You must provide exactly one path argument.\n') + exit(1) + +listFilePath = os.path.abspath(pathArgs[0]) +for suffix, include, cmakeOpts in ALL_MAPS: + subdir = "build-%s" % suffix + if not os.path.exists(subdir): + os.mkdir(subdir) + os.chdir(subdir) + print("Configuring in %s..." % subdir) + with open('junction_userconfig.h.in', 'w') as f: + f.write('#define JUNCTION_IMPL_MAPADAPTER_PATH "%s"\n' % include) + + userConfigCMakePath = os.path.abspath('junction_userconfig.h.in') + if os.sep != '/': + userConfigCMakePath = userConfigCMakePath.replace(os.sep, '/') + if subprocess.call([CMAKE, listFilePath, '-DCMAKE_INSTALL_PREFIX=TestAllMapsInstallFolder', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', '-DTURF_USE_DLMALLOC=1', '-DTURF_DLMALLOC_FAST_STATS=1', + '-DJUNCTION_USERCONFIG=%s' % userConfigCMakePath] + passThroughArgs + cmakeOpts) == 0: + subprocess.check_call([CMAKE, '--build', '.', '--target', 'install', '--config', 'RelWithDebInfo']) + print('Running in %s...' % subdir) + results = subprocess.check_output([os.path.join('TestAllMapsInstallFolder', 'bin', 'MapMemoryBench')]) + with open('results.txt', 'wb') as f: + f.write(results) + os.chdir("..") diff --git a/samples/MapPerformanceTests/CMakeLists.txt b/samples/MapPerformanceTests/CMakeLists.txt new file mode 100644 index 0000000..9838328 --- /dev/null +++ b/samples/MapPerformanceTests/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8.5) + +get_filename_component(SAMPLE_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # CMAKE_CONFIGURATION_TYPES only reliable if set before project(), and not from an include file + set(CMAKE_CONFIGURATION_TYPES "Debug;RelWithAsserts;RelWithDebInfo" CACHE INTERNAL "Build configs") + project(${SAMPLE_NAME}) +endif() + +include(../../cmake/AddSample.cmake) diff --git a/samples/MapPerformanceTests/MapPerformanceTests.cpp b/samples/MapPerformanceTests/MapPerformanceTests.cpp new file mode 100644 index 0000000..4c9b91e --- /dev/null +++ b/samples/MapPerformanceTests/MapPerformanceTests.cpp @@ -0,0 +1,301 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace turf::intTypes; +typedef junction::extra::MapAdapter MapAdapter; + +static const ureg NumKeysPerThread = 16384; +static const ureg DefaultReadsPerWrite = 19; +static const ureg DefaultItersPerChunk = 128; +static const ureg DefaultChunks = 10; +static const u32 Prime = 0x4190ab09; + +class Delay { +private: + turf::extra::Random m_rand; + u32 m_threshold; + +public: + Delay(float ratio) { + m_threshold = u32(double(0xffffffffu) * ratio); + } + + void delay(ureg& workUnits) { + for (;;) { + volatile ureg v = m_rand.next32(); + workUnits++; + if (v <= m_threshold) + break; + } + } +}; + +struct SharedState { + MapAdapter& adapter; + MapAdapter::Map* map; + ureg numKeysPerThread; + float delayFactor; + ureg numThreads; + ureg readsPerWrite; + ureg itersPerChunk; + turf::extra::SpinKicker spinKicker; + turf::Atomic doneFlag; + + SharedState(MapAdapter& adapter, ureg numThreads, ureg numKeysPerThread, ureg readsPerWrite, ureg itersPerChunk) + : adapter(adapter), map(NULL), numKeysPerThread(numKeysPerThread), numThreads(numThreads), readsPerWrite(readsPerWrite), itersPerChunk(itersPerChunk) { + delayFactor = 0.5f; + doneFlag.storeNonatomic(0); + } +}; + +class ThreadState { +public: + SharedState& m_shared; + MapAdapter::ThreadContext m_threadCtx; + ureg m_threadIndex; + u32 m_rangeLo; + u32 m_rangeHi; + + u32 m_addIndex; + u32 m_removeIndex; + + struct Stats { + ureg workUnitsDone; + ureg mapOpsDone; + double duration; + + Stats() { + workUnitsDone = 0; + mapOpsDone = 0; + duration = 0; + } + + Stats& operator+=(const Stats& other) { + workUnitsDone += other.workUnitsDone; + mapOpsDone += other.mapOpsDone; + duration += other.duration; + return *this; + } + + bool operator<(const Stats& other) const { + return duration < other.duration; + } + }; + + Stats m_stats; + + ThreadState(SharedState& shared, ureg threadIndex, u32 rangeLo, u32 rangeHi) : m_shared(shared), m_threadCtx(shared.adapter, threadIndex) { + m_threadIndex = threadIndex; + m_rangeLo = rangeLo; + m_rangeHi = rangeHi; + m_addIndex = rangeLo; + m_removeIndex = rangeLo; + } + + void registerThread() { + m_threadCtx.registerThread(); + } + + void unregisterThread() { + m_threadCtx.unregisterThread(); + } + + void initialPopulate() { + TURF_ASSERT(m_addIndex == m_removeIndex); + MapAdapter::Map *map = m_shared.map; + for (ureg i = 0; i < m_shared.numKeysPerThread; i++) { + u32 key = m_addIndex * Prime; + map->insert(key, (void*) (key & ~uptr(3))); + if (++m_addIndex == m_rangeHi) + m_addIndex = m_rangeLo; + } + } + + void run() { + MapAdapter::Map *map = m_shared.map; + turf::CPUTimer::Converter converter; + Delay delay(m_shared.delayFactor); + Stats stats; + ureg lookupIndex = m_rangeLo; + ureg remaining = m_shared.itersPerChunk; + if (m_threadIndex == 0) + m_shared.spinKicker.kick(m_shared.numThreads - 1); + else { + remaining = ~u32(0); + m_shared.spinKicker.waitForKick(); + } + + // --------- + turf::CPUTimer::Point start = turf::CPUTimer::get(); + for (; remaining > 0; remaining--) { + // Add + delay.delay(stats.workUnitsDone); + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + u32 key = m_addIndex * Prime; + if (key >= 2) { + map->insert(key, (void*) uptr(key)); + stats.mapOpsDone++; + } + if (++m_addIndex == m_rangeHi) + m_addIndex = m_rangeLo; + + // Lookup + if (s32(lookupIndex - m_removeIndex) < 0) + lookupIndex = m_removeIndex; + for (ureg l = 0; l < m_shared.readsPerWrite; l++) { + delay.delay(stats.workUnitsDone); + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + key = lookupIndex * Prime; + if (key >= 2) { + volatile void* value = map->get(key); + TURF_UNUSED(value); + stats.mapOpsDone++; + } + if (++lookupIndex == m_rangeHi) + lookupIndex = m_rangeLo; + if (lookupIndex == m_addIndex) + lookupIndex = m_removeIndex; + } + + // Remove + delay.delay(stats.workUnitsDone); + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + key = m_removeIndex * Prime; + if (key >= 2) { + map->erase(key); + stats.mapOpsDone++; + } + if (++m_removeIndex == m_rangeHi) + m_removeIndex = m_rangeLo; + + // Lookup + if (s32(lookupIndex - m_removeIndex) < 0) + lookupIndex = m_removeIndex; + for (ureg l = 0; l < m_shared.readsPerWrite; l++) { + delay.delay(stats.workUnitsDone); + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + key = lookupIndex * Prime; + if (key >= 2) { + volatile void* value = map->get(key); + TURF_UNUSED(value); + stats.mapOpsDone++; + } + if (++lookupIndex == m_rangeHi) + lookupIndex = m_rangeLo; + if (lookupIndex == m_addIndex) + lookupIndex = m_removeIndex; + } + } + if (m_threadIndex == 0) + m_shared.doneFlag.store(1, turf::Relaxed); + m_threadCtx.update(); + turf::CPUTimer::Point end = turf::CPUTimer::get(); + // --------- + + stats.duration = converter.toSeconds(end - start); + m_stats = stats; + } +}; + +static const turf::extra::Option Options[] = { + { "readsPerWrite", 'r', true, "number of reads per write" }, + { "itersPerChunk", 'i', true, "number of iterations per chunk" }, + { "chunks", 'c', true, "number of chunks to execute" }, + { "keepChunkFraction", 'k', true, "threshold fraction of chunk timings to keep" }, +}; + +int main(int argc, const char** argv) { + turf::extra::Options options(Options, TURF_STATIC_ARRAY_SIZE(Options)); + options.parse(argc, argv); + ureg readsPerWrite = options.getInteger("readsPerWrite", DefaultReadsPerWrite); + ureg itersPerChunk = options.getInteger("itersPerChunk", DefaultItersPerChunk); + ureg chunks = options.getInteger("chunks", DefaultChunks); + double keepChunkFraction = options.getDouble("keepChunkFraction", 1.0); + + turf::extra::JobDispatcher dispatcher; + ureg numThreads = dispatcher.getNumPhysicalCores(); + MapAdapter adapter(numThreads); + + // Create shared state and register threads + SharedState shared(adapter, numThreads, NumKeysPerThread, readsPerWrite, itersPerChunk); + std::vector threads; + threads.reserve(numThreads); + for (ureg t = 0; t < numThreads; t++) { + u32 rangeLo = 0xffffffffu / numThreads * t + 1; + u32 rangeHi = 0xffffffffu / numThreads * (t + 1) + 1; + threads.emplace_back(shared, t, rangeLo, rangeHi); + } + dispatcher.kickMulti(&ThreadState::registerThread, &threads[0], threads.size()); + + { + // Create the map + MapAdapter::Map map(MapAdapter::getInitialCapacity(numThreads * NumKeysPerThread)); + shared.map = ↦ + dispatcher.kickMulti(&ThreadState::initialPopulate, &threads[0], threads.size()); + + printf("{\n"); + printf("'mapType': '%s',\n", MapAdapter::MapName); + printf("'readsPerWrite': %d,\n", (int) readsPerWrite); + printf("'itersPerChunk': %d,\n", (int) itersPerChunk); + printf("'chunks': %d,\n", (int) chunks); + printf("'keepChunkFraction': %f,\n", keepChunkFraction); + printf("'labels': ('delayFactor', 'workUnitsDone', 'mapOpsDone', 'totalTime'),\n"), + printf("'points': [\n"); + for (float delayFactor = 1.f; delayFactor >= 0.0005f; delayFactor *= 0.95f) + { + shared.delayFactor = delayFactor; + + std::vector kickTotals; + for (ureg c = 0; c < chunks; c++) { + shared.doneFlag.storeNonatomic(false); + dispatcher.kickMulti(&ThreadState::run, &threads[0], threads.size()); + + ThreadState::Stats kickTotal; + for (ureg t = 0; t < numThreads; t++) + kickTotal += threads[t].m_stats; + kickTotals.push_back(kickTotal); + } + + std::sort(kickTotals.begin(), kickTotals.end()); + ThreadState::Stats totals; + for (ureg t = 0; t < ureg(kickTotals.size() * keepChunkFraction); t++) { + totals += kickTotals[t]; + } + + printf(" (%f, %d, %d, %f),\n", + shared.delayFactor, + int(totals.workUnitsDone), + int(totals.mapOpsDone), + totals.duration); + } + printf("],\n"); + printf("}\n"); + + shared.map = NULL; + } + + dispatcher.kickMulti(&ThreadState::unregisterThread, &threads[0], threads.size()); + return 0; +} diff --git a/samples/MapPerformanceTests/RenderGraphs.py b/samples/MapPerformanceTests/RenderGraphs.py new file mode 100644 index 0000000..3479f13 --- /dev/null +++ b/samples/MapPerformanceTests/RenderGraphs.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python +import os +import cairo +import math +import glob +import collections + + +def colorTuple(h): + return (int(h[0:2], 16) / 255.0, int(h[2:4], 16) / 255.0, int(h[4:6], 16) / 255.0) + +NULL_MAP = 'null' +ALL_MAPS = [ + ('folly', colorTuple('606080')), + ('tervel', colorTuple('9090b0')), + ('stdmap', colorTuple('606080')), + ('nbds', colorTuple('9090b0')), + ('michael', colorTuple('606080')), + ('tbb', colorTuple('9090b0')), + ('linear', colorTuple('ff4040')), + ('grampa', colorTuple('ff4040')), + ('leapfrog', colorTuple('ff4040')), +] + +#--------------------------------------------------- +# Cairo drawing helpers +#--------------------------------------------------- +def createScaledFont(family, size, slant=cairo.FONT_SLANT_NORMAL, weight=cairo.FONT_WEIGHT_NORMAL): + """ Simple helper function to create a cairo ScaledFont. """ + face = cairo.ToyFontFace(family, slant, weight) + DEFAULT_FONT_OPTIONS = cairo.FontOptions() + DEFAULT_FONT_OPTIONS.set_antialias(cairo.ANTIALIAS_SUBPIXEL) + return cairo.ScaledFont(face, cairo.Matrix(xx=size, yy=size), cairo.Matrix(), DEFAULT_FONT_OPTIONS) + +def fillAlignedText(cr, x, y, scaledFont, text, alignment = 0): + """ Draw some aligned text at the specified co-ordinates. + alignment = 0: left-justify + alignment = 0.5: center + alignment = 1: right-justify """ + ascent, descent = scaledFont.extents()[:2] + x_bearing, y_bearing, width, height = scaledFont.text_extents(text)[:4] + with Saved(cr): + cr.set_scaled_font(scaledFont) + cr.move_to(math.floor(x + 0.5 - width * alignment), math.floor(y + 0.5)) + cr.text_path(text) + cr.fill() + +class Saved(): + """ Preserve cairo state inside the scope of a with statement. """ + def __init__(self, cr): + self.cr = cr + def __enter__(self): + self.cr.save() + return self.cr + def __exit__(self, type, value, traceback): + self.cr.restore() + + +#--------------------------------------------------- +# AxisAttribs +#--------------------------------------------------- +class AxisAttribs: + """ Describes one axis on the graph. Can be linear or logarithmic. """ + + def __init__(self, size, min, max, step, logarithmic = False, labeler = lambda x: str(int(x + 0.5))): + self.size = float(size) + self.logarithmic = logarithmic + self.labeler = labeler + self.toAxis = lambda x: math.log(x) if logarithmic else float(x) + self.fromAxis = lambda x: math.exp(x) if logarithmic else float(x) + self.min = self.toAxis(min) + self.max = self.toAxis(max) + self.step = self.toAxis(step) + + def setMinMax(self, min, max): + self.min = self.toAxis(min) + self.max = self.toAxis(max) + + def mapAxisValue(self, x): + """ Maps x to a point along the axis. """ + return (self.toAxis(x) - self.min) / (self.max - self.min) * self.size + + def iterLabels(self): + """ Helper to iterate through all the tick marks along the axis. """ + lo = int(math.floor(self.min / self.step + 1 - 1e-9)) + hi = int(math.floor(self.max / self.step + 1e-9)) + for i in xrange(lo, hi + 1): + value = i * self.step + if self.min == 0 and i == 0: + continue + yield self.mapAxisValue(self.fromAxis(value)), self.labeler(self.fromAxis(value)) + + +#--------------------------------------------------- +# Graph +#--------------------------------------------------- +def makeNamedTuples(results, typeName = 'Point', labels = 'labels', points = 'points'): + namedTupleType = collections.namedtuple(typeName, results[labels]) + numLabels = len(results[labels]) + return [namedTupleType(*p[:numLabels]) for p in results[points]] + +class Curve: + def __init__(self, name, points, color): + self.name = name + self.points = points + self.color = color + +class Graph: + """ Renders a graph. """ + + def __init__(self, xAttribs, yAttribs): + self.xAttribs = xAttribs + self.yAttribs = yAttribs + self.curves = [] + self.xMin = None + self.xMax = None + + def addCurve(self, curve): + self.curves.append(curve) + xMin = min(x for x, y in curve.points) + if self.xMin is None or xMin < self.xMin: + self.xMin = xMin + xMax = max(x for x, y in curve.points) + if self.xMax is None or xMax > self.xMax: + self.xMax = xMax + + def renderTo(self, fileName): + xAttribs = self.xAttribs + yAttribs = self.yAttribs + xAttribs.setMinMax(self.xMin, self.xMax) + + # Create the image surface and cairo context + surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, 150 + int(xAttribs.size + 0.5), 65 + int(yAttribs.size + 0.5)) + cr = cairo.Context(surface) + cr.set_source_rgb(1, 1, 1) + cr.paint() + cr.set_miter_limit(1.414) + cr.translate(58, 11 + yAttribs.size) + + # Draw axes + labelFont = createScaledFont('Arial', 11) + with Saved(cr): + cr.set_line_width(1) + cr.set_source_rgb(.4, .4, .4) + + # Horizontal axis + cr.move_to(0, -0.5) + cr.rel_line_to(xAttribs.size + 1, 0) + for pos, label in xAttribs.iterLabels(): # Tick marks + x = math.floor(pos + 0.5) + 0.5 + cr.move_to(x, -1) + cr.rel_line_to(0, 4) + cr.stroke() + for pos, label in xAttribs.iterLabels(): # Labels + x = math.floor(pos + 0.5) + with Saved(cr): + cr.translate(x - 1, 5) + cr.rotate(-math.pi / 4) + fillAlignedText(cr, 0, 6, labelFont, label, 1) + + # Vertical axis + cr.set_source_rgb(*colorTuple('f0f0f0')) + for pos, label in yAttribs.iterLabels(): # Background lines + if label == '0': + continue + y = -math.floor(pos + 0.5) - 0.5 + cr.move_to(1, y) + cr.rel_line_to(xAttribs.size + 1, 0) + cr.stroke() + cr.set_source_rgb(.4, .4, .4) + cr.move_to(0.5, 0) + cr.rel_line_to(0, -yAttribs.size - 0.5) + if False: + for pos, label in yAttribs.iterLabels(): # Tick marks + if label == '0': + continue + y = -math.floor(pos + 0.5) - 0.5 + cr.move_to(1, y) + cr.rel_line_to(-4, 0) + cr.stroke() + for pos, label in yAttribs.iterLabels(): # Labels + if label == '0': + continue + fillAlignedText(cr, -4, -pos + 4, labelFont, label, 1) + + with Saved(cr): + x = xAttribs.size - 70.5 + y = -234.5 + cr.rectangle(x, y, 120, 82) + cr.set_source_rgb(*colorTuple('ffffff')) + cr.fill() + cr.set_source_rgb(*colorTuple('f0f0f0')) + cr.rectangle(x, y, 120, 82) + cr.set_line_width(1) + cr.stroke() + + # Draw curves + for cn, curve in enumerate(self.curves): + points = curve.points + width = 2.5 + color = curve.color + with Saved(cr): + cr.set_line_width(width) + cr.set_source_rgba(*color) + if cn in [1, 3, 5]: + cr.set_dash([10, 1]) + with Saved(cr): + cr.rectangle(0, 5, xAttribs.size, -yAttribs.size - 15) + cr.clip() + x, y = points[0] + cr.move_to(xAttribs.mapAxisValue(x), -yAttribs.mapAxisValue(y)) + for x, y in points[1:]: + cr.line_to(xAttribs.mapAxisValue(x) + 0.5, -yAttribs.mapAxisValue(y) - 0.5) + x = xAttribs.size - 40 + y = -220 + cn * 12 + cr.move_to(x - 4.5, y - 4.5) + cr.rel_line_to(-21, 0) + cr.stroke() + + # Label + labelFont = createScaledFont('Arial', 11) + label = curve.name + #x, y = points[-1] + #fillAlignedText(cr, xAttribs.mapAxisValue(x) + 3, -yAttribs.mapAxisValue(y) + 4, labelFont, label, 0) + x = xAttribs.size - 40 + y = -220 + cn * 12 + fillAlignedText(cr, x, y, labelFont, label, 0) + + # Draw axis names + cr.set_source_rgb(0, 0, 0) + axisFont = createScaledFont('Helvetica', 16, weight=cairo.FONT_WEIGHT_BOLD) + with Saved(cr): + cr.translate(-44, -yAttribs.size / 2.0) + cr.rotate(-math.pi / 2) + fillAlignedText(cr, 0, 0, axisFont, 'CPU Time Spent in Map', 0.5) + fillAlignedText(cr, xAttribs.size / 2.0, 50, axisFont, 'Interval Between Map Operations', 0.5) + + # Save PNG file + surface.write_to_png(fileName) + + +#--------------------------------------------------- +# main +#--------------------------------------------------- +def formatTime(x): + if x < 0.9e-6: + return '%d ns' % (x * 1e9 + 0.5) + elif x < 0.9e-3: + return '%d us' % (x * 1e6 + 0.5) # FIXME: Micro symbol + if x < 0.9: + return '%d ms' % (x * 1e3 + 0.5) + else: + return '%d s' % (x + 0.5) + +graph = Graph(AxisAttribs(550, 1e-9, 10000e-9, 10, True, formatTime), + AxisAttribs(240, 0, 1, 0.1, False, lambda x: '%d%%' % int(x * 100 + 0.5))) + +nullResultFn = 'build-%s/results.txt' % NULL_MAP +nullResults = eval(open(nullResultFn, 'r').read()) +nullPoints = makeNamedTuples(nullResults) + +for suffix, color in ALL_MAPS: + resultsPath = 'build-%s/results.txt' % suffix + if os.path.exists(resultsPath): + with open(resultsPath, 'r') as f: + results = eval(f.read()) + dataPoints = makeNamedTuples(results) + def makeGraphPoint(nullPt, pt): + wuPerOp = float(nullPt.workUnitsDone) / nullPt.mapOpsDone + timePerWU = nullPt.totalTime / nullPt.workUnitsDone + timeBetweenOps = wuPerOp * timePerWU + mapTime = pt.totalTime - pt.workUnitsDone * timePerWU + return (timeBetweenOps, mapTime / pt.totalTime) + graphPoints = [makeGraphPoint(*pair) for pair in zip(nullPoints, dataPoints)] + def smooth(pts): + result = [] + for i in xrange(len(pts) - 1): + x0, y0 = pts[i] + x1, y1 = pts[i + 1] + result += [(0.75*x0 + 0.25*x1, 0.75*y0 + 0.25*y1), + (0.25*x0 + 0.75*x1, 0.25*y0 + 0.75*y1)] + return result + def smooth2(pts): + result = [] + for i in xrange(len(pts) - 1): + x0, y0 = pts[i] + x1, y1 = pts[i + 1] + result += [(0.5*x0 + 0.5*x1, 0.5*y0 + 0.5*y1)] + return result + + graphPoints = smooth2(graphPoints) + graphPoints = smooth2(graphPoints) + graphPoints = smooth2(graphPoints) + graphPoints = smooth2(graphPoints) + graphPoints = smooth2(graphPoints) + graphPoints = smooth2(graphPoints) + graphPoints = smooth(graphPoints) + graphPoints = smooth2(graphPoints) + graph.addCurve(Curve(results['mapType'], graphPoints, color)) + +graph.renderTo('out.png') diff --git a/samples/MapPerformanceTests/TestAllMaps.py b/samples/MapPerformanceTests/TestAllMaps.py new file mode 100644 index 0000000..1e76984 --- /dev/null +++ b/samples/MapPerformanceTests/TestAllMaps.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +import os +import subprocess +import sys + +# Default CMake command is just 'cmake' but you can override it by setting +# the CMAKE environment variable: +CMAKE = os.getenv('CMAKE', 'cmake') + +ALL_MAPS = [ + ('null', 'junction/extra/impl/MapAdapter_Null.h', [], ['-i256', '-c10']), + ('michael', 'junction/extra/impl/MapAdapter_CDS_Michael.h', ['-DJUNCTION_WITH_CDS=1', '-DTURF_WITH_EXCEPTIONS=1'], ['-i256', '-c10']), + ('linear', 'junction/extra/impl/MapAdapter_Linear.h', [], ['-i256', '-c10']), + ('leapfrog', 'junction/extra/impl/MapAdapter_LeapFrog.h', [], ['-i256', '-c10']), + ('grampa', 'junction/extra/impl/MapAdapter_Grampa.h', [], ['-i256', '-c10']), + ('stdmap', 'junction/extra/impl/MapAdapter_StdMap.h', [], ['-i256', '-c10']), + ('folly', 'junction/extra/impl/MapAdapter_Folly.h', ['-DJUNCTION_WITH_FOLLY=1', '-DTURF_WITH_EXCEPTIONS=1'], ['-i256', '-c1']), + ('nbds', 'junction/extra/impl/MapAdapter_NBDS.h', ['-DJUNCTION_WITH_NBDS=1'], ['-i256', '-c10']), + ('tbb', 'junction/extra/impl/MapAdapter_TBB.h', ['-DJUNCTION_WITH_TBB=1'], ['-i256', '-c10']), + ('tervel', 'junction/extra/impl/MapAdapter_Tervel.h', ['-DJUNCTION_WITH_TERVEL=1'], ['-i256', '-c1']), +] + +# Scan arguments for path to CMakeLists.txt and args to pass through. +passThroughArgs = [] +pathArgs = [] +for arg in sys.argv[1:]: + if arg.startswith('-'): + passThroughArgs.append(arg) + else: + pathArgs.append(arg) +if len(pathArgs) != 1: + sys.stderr.write('You must provide exactly one path argument.\n') + exit(1) + +listFilePath = os.path.abspath(pathArgs[0]) +for suffix, include, cmakeOpts, runtimeOpts in ALL_MAPS: + subdir = 'build-%s' % suffix + if not os.path.exists(subdir): + os.mkdir(subdir) + os.chdir(subdir) + print('Configuring in %s...' % subdir) + with open('junction_userconfig.h.in', 'w') as f: + f.write('#define JUNCTION_IMPL_MAPADAPTER_PATH "%s"\n' % include) + userConfigCMakePath = os.path.abspath('junction_userconfig.h.in') + if os.sep != '/': + userConfigCMakePath = userConfigCMakePath.replace(os.sep, '/') + if subprocess.call([CMAKE, listFilePath, '-DCMAKE_INSTALL_PREFIX=TestAllMapsInstallFolder', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', '-DTURF_USE_DLMALLOC=1', '-DTURF_DLMALLOC_FAST_STATS=1', + '-DJUNCTION_USERCONFIG=%s' % userConfigCMakePath] + passThroughArgs + cmakeOpts) == 0: + subprocess.check_call([CMAKE, '--build', '.', '--target', 'install', '--config', 'RelWithDebInfo']) + print('Running in %s...' % subdir) + results = subprocess.check_output([os.path.join('TestAllMapsInstallFolder', 'bin', 'MapPerformanceTests')] + runtimeOpts) + with open('results.txt', 'wb') as f: + f.write(results) + os.chdir('..') diff --git a/samples/MapScalabilityTests/CMakeLists.txt b/samples/MapScalabilityTests/CMakeLists.txt new file mode 100644 index 0000000..9838328 --- /dev/null +++ b/samples/MapScalabilityTests/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8.5) + +get_filename_component(SAMPLE_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # CMAKE_CONFIGURATION_TYPES only reliable if set before project(), and not from an include file + set(CMAKE_CONFIGURATION_TYPES "Debug;RelWithAsserts;RelWithDebInfo" CACHE INTERNAL "Build configs") + project(${SAMPLE_NAME}) +endif() + +include(../../cmake/AddSample.cmake) diff --git a/samples/MapScalabilityTests/MapScalabilityTests.cpp b/samples/MapScalabilityTests/MapScalabilityTests.cpp new file mode 100644 index 0000000..8f84110 --- /dev/null +++ b/samples/MapScalabilityTests/MapScalabilityTests.cpp @@ -0,0 +1,278 @@ +/*------------------------------------------------------------------------ + Junction: Concurrent data structures in C++ + Copyright (c) 2016 Jeff Preshing + + Distributed under the Simplified BSD License. + Original location: https://github.com/preshing/junction + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the LICENSE file for more information. +------------------------------------------------------------------------*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace turf::intTypes; +typedef junction::extra::MapAdapter MapAdapter; + +static const ureg NumKeysPerThread = 2000; +static const ureg DefaultReadsPerWrite = 4; +static const ureg DefaultItersPerChunk = 10000; +static const ureg DefaultChunks = 200; +static const u32 Prime = 0x4190ab09; + +struct SharedState { + MapAdapter& adapter; + MapAdapter::Map* map; + ureg numKeysPerThread; + ureg numThreads; + ureg readsPerWrite; + ureg itersPerChunk; + turf::extra::SpinKicker spinKicker; + turf::Atomic doneFlag; + + SharedState(MapAdapter& adapter, ureg numKeysPerThread, ureg readsPerWrite, ureg itersPerChunk) + : adapter(adapter), map(NULL), numKeysPerThread(numKeysPerThread), readsPerWrite(readsPerWrite), itersPerChunk(itersPerChunk) { + doneFlag.storeNonatomic(0); + numThreads = 0; + } +}; + +class ThreadState { +public: + SharedState& m_shared; + MapAdapter::ThreadContext m_threadCtx; + ureg m_threadIndex; + u32 m_rangeLo; + u32 m_rangeHi; + + u32 m_addIndex; + u32 m_removeIndex; + + struct Stats { + ureg mapOpsDone; + double duration; + + Stats() { + mapOpsDone = 0; + duration = 0; + } + + Stats& operator+=(const Stats& other) { + mapOpsDone += other.mapOpsDone; + duration += other.duration; + return *this; + } + + bool operator<(const Stats& other) const { + return duration < other.duration; + } + }; + + Stats m_stats; + + ThreadState(SharedState& shared, ureg threadIndex, u32 rangeLo, u32 rangeHi) : m_shared(shared), m_threadCtx(shared.adapter, threadIndex) { + m_threadIndex = threadIndex; + m_rangeLo = rangeLo; + m_rangeHi = rangeHi; + m_addIndex = rangeLo; + m_removeIndex = rangeLo; + } + + void registerThread() { + m_threadCtx.registerThread(); + } + + void unregisterThread() { + m_threadCtx.unregisterThread(); + } + + void initialPopulate() { + TURF_ASSERT(m_addIndex == m_removeIndex); + MapAdapter::Map *map = m_shared.map; + for (ureg i = 0; i < m_shared.numKeysPerThread; i++) { + u32 key = m_addIndex * Prime; + if (key >= 2) + map->insert(key, (void*) uptr(key)); + if (++m_addIndex == m_rangeHi) + m_addIndex = m_rangeLo; + } + } + + void run() { + MapAdapter::Map *map = m_shared.map; + turf::CPUTimer::Converter converter; + Stats stats; + ureg lookupIndex = m_rangeLo; + ureg remaining = m_shared.itersPerChunk; + if (m_threadIndex == 0) + m_shared.spinKicker.kick(m_shared.numThreads - 1); + else { + remaining = ~u32(0); + m_shared.spinKicker.waitForKick(); + } + + // --------- + turf::CPUTimer::Point start = turf::CPUTimer::get(); + for (; remaining > 0; remaining--) { + // Add + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + u32 key = m_addIndex * Prime; + if (key >= 2) { + map->insert(key, (void*) uptr(key)); + stats.mapOpsDone++; + } + if (++m_addIndex == m_rangeHi) + m_addIndex = m_rangeLo; + + // Lookup + if (s32(lookupIndex - m_removeIndex) < 0) + lookupIndex = m_removeIndex; + for (ureg l = 0; l < m_shared.readsPerWrite; l++) { + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + key = lookupIndex * Prime; + if (key >= 2) { + volatile void* value = map->get(key); + TURF_UNUSED(value); + stats.mapOpsDone++; + } + if (++lookupIndex == m_rangeHi) + lookupIndex = m_rangeLo; + if (lookupIndex == m_addIndex) + lookupIndex = m_removeIndex; + } + + // Remove + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + key = m_removeIndex * Prime; + if (key >= 2) { + map->erase(key); + stats.mapOpsDone++; + } + if (++m_removeIndex == m_rangeHi) + m_removeIndex = m_rangeLo; + + // Lookup + if (s32(lookupIndex - m_removeIndex) < 0) + lookupIndex = m_removeIndex; + for (ureg l = 0; l < m_shared.readsPerWrite; l++) { + if (m_shared.doneFlag.load(turf::Relaxed)) + break; + key = lookupIndex * Prime; + if (key >= 2) { + volatile void* value = map->get(key); + TURF_UNUSED(value); + stats.mapOpsDone++; + } + if (++lookupIndex == m_rangeHi) + lookupIndex = m_rangeLo; + if (lookupIndex == m_addIndex) + lookupIndex = m_removeIndex; + } + } + if (m_threadIndex == 0) + m_shared.doneFlag.store(1, turf::Relaxed); + m_threadCtx.update(); + turf::CPUTimer::Point end = turf::CPUTimer::get(); + // --------- + + stats.duration = converter.toSeconds(end - start); + m_stats = stats; + } +}; + +static const turf::extra::Option Options[] = { + { "readsPerWrite", 'r', true, "number of reads per write" }, + { "itersPerChunk", 'i', true, "number of iterations per chunk" }, + { "chunks", 'c', true, "number of chunks to execute" }, + { "keepChunkFraction", 'k', true, "threshold fraction of chunk timings to keep" }, +}; + +int main(int argc, const char** argv) { + turf::extra::Options options(Options, TURF_STATIC_ARRAY_SIZE(Options)); + options.parse(argc, argv); + ureg readsPerWrite = options.getInteger("readsPerWrite", DefaultReadsPerWrite); + ureg itersPerChunk = options.getInteger("itersPerChunk", DefaultItersPerChunk); + ureg chunks = options.getInteger("chunks", DefaultChunks); + double keepChunkFraction = options.getDouble("keepChunkFraction", 1.0); + + turf::extra::JobDispatcher dispatcher; + ureg numCores = dispatcher.getNumPhysicalCores(); + TURF_ASSERT(numCores > 0); + MapAdapter adapter(numCores); + + // Create shared state and register first thread + SharedState shared(adapter, NumKeysPerThread, readsPerWrite, itersPerChunk); + std::vector threads; + threads.reserve(numCores); + for (ureg t = 0; t < numCores; t++) { + u32 rangeLo = 0xffffffffu / numCores * t + 1; + u32 rangeHi = 0xffffffffu / numCores * (t + 1) + 1; + threads.emplace_back(shared, t, rangeLo, rangeHi); + } + dispatcher.kickOne(0, &ThreadState::registerThread, threads[0]); + + { + // Create the map and populate it entirely from main thread + MapAdapter::Map map(MapAdapter::getInitialCapacity(numCores * NumKeysPerThread)); + shared.map = ↦ + for (ureg t = 0; t < numCores; t++) { + threads[t].initialPopulate(); + } + + printf("{\n"); + printf("'mapType': '%s',\n", MapAdapter::MapName); + printf("'population': %d,\n", (int) (numCores * NumKeysPerThread)); + printf("'readsPerWrite': %d,\n", (int) readsPerWrite); + printf("'itersPerChunk': %d,\n", (int) itersPerChunk); + printf("'chunks': %d,\n", (int) chunks); + printf("'keepChunkFraction': %f,\n", keepChunkFraction); + printf("'labels': ('numThreads', 'mapOpsDone', 'totalTime'),\n"), + printf("'points': [\n"); + for (shared.numThreads = 1; shared.numThreads <= numCores; shared.numThreads++) { + if (shared.numThreads > 1) { + // Spawn and register a new thread + dispatcher.kickOne(shared.numThreads - 1, &ThreadState::registerThread, threads[shared.numThreads - 1]); + } + + std::vector kickTotals; + for (ureg c = 0; c < chunks; c++) { + shared.doneFlag.storeNonatomic(false); + dispatcher.kickMulti(&ThreadState::run, &threads[0], shared.numThreads); + + ThreadState::Stats kickTotal; + for (ureg t = 0; t < shared.numThreads; t++) + kickTotal += threads[t].m_stats; + kickTotals.push_back(kickTotal); + } + + std::sort(kickTotals.begin(), kickTotals.end()); + ThreadState::Stats totals; + for (ureg t = 0; t < ureg(kickTotals.size() * keepChunkFraction); t++) { + totals += kickTotals[t]; + } + + printf(" (%d, %d, %f),\n", + int(shared.numThreads), + int(totals.mapOpsDone), + totals.duration); + } + printf("],\n"); + printf("}\n"); + + shared.map = NULL; + } + + dispatcher.kickMulti(&ThreadState::unregisterThread, &threads[0], threads.size()); + return 0; +} diff --git a/samples/MapScalabilityTests/RenderGraphs.py b/samples/MapScalabilityTests/RenderGraphs.py new file mode 100644 index 0000000..9b21d85 --- /dev/null +++ b/samples/MapScalabilityTests/RenderGraphs.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python +import os +import cairo +import math +import glob +import collections + + +def colorTuple(h): + return (int(h[0:2], 16) / 255.0, int(h[2:4], 16) / 255.0, int(h[4:6], 16) / 255.0) + +ALL_MAPS = [ + ('folly', colorTuple('606080')), + ('tervel', colorTuple('9090b0')), + ('stdmap', colorTuple('606080')), + ('nbds', colorTuple('9090b0')), + ('michael', colorTuple('606080')), + ('tbb', colorTuple('9090b0')), + ('linear', colorTuple('ff4040')), + ('grampa', colorTuple('ff4040')), + ('leapfrog', colorTuple('ff4040')), +] + +#--------------------------------------------------- +# Cairo drawing helpers +#--------------------------------------------------- +def createScaledFont(family, size, slant=cairo.FONT_SLANT_NORMAL, weight=cairo.FONT_WEIGHT_NORMAL): + """ Simple helper function to create a cairo ScaledFont. """ + face = cairo.ToyFontFace(family, slant, weight) + DEFAULT_FONT_OPTIONS = cairo.FontOptions() + DEFAULT_FONT_OPTIONS.set_antialias(cairo.ANTIALIAS_SUBPIXEL) + return cairo.ScaledFont(face, cairo.Matrix(xx=size, yy=size), cairo.Matrix(), DEFAULT_FONT_OPTIONS) + +def fillAlignedText(cr, x, y, scaledFont, text, alignment = 0): + """ Draw some aligned text at the specified co-ordinates. + alignment = 0: left-justify + alignment = 0.5: center + alignment = 1: right-justify """ + ascent, descent = scaledFont.extents()[:2] + x_bearing, y_bearing, width, height = scaledFont.text_extents(text)[:4] + with Saved(cr): + cr.set_scaled_font(scaledFont) + cr.move_to(math.floor(x + 0.5 - width * alignment), math.floor(y + 0.5)) + cr.text_path(text) + cr.fill() + +class Saved(): + """ Preserve cairo state inside the scope of a with statement. """ + def __init__(self, cr): + self.cr = cr + def __enter__(self): + self.cr.save() + return self.cr + def __exit__(self, type, value, traceback): + self.cr.restore() + + +#--------------------------------------------------- +# AxisAttribs +#--------------------------------------------------- +class AxisAttribs: + """ Describes one axis on the graph. Can be linear or logarithmic. """ + + def __init__(self, size, min, max, step, logarithmic = False, labeler = lambda x: str(int(x + 0.5))): + self.size = float(size) + self.logarithmic = logarithmic + self.labeler = labeler + self.toAxis = lambda x: math.log(x) if logarithmic else float(x) + self.fromAxis = lambda x: math.exp(x) if logarithmic else float(x) + self.min = self.toAxis(min) + self.max = self.toAxis(max) + self.step = self.toAxis(step) + + def setMinMax(self, min, max): + self.min = self.toAxis(min) + self.max = self.toAxis(max) + + def mapAxisValue(self, x): + """ Maps x to a point along the axis. """ + return (self.toAxis(x) - self.min) / (self.max - self.min) * self.size + + def iterLabels(self): + """ Helper to iterate through all the tick marks along the axis. """ + lo = int(math.floor(self.min / self.step + 1 - 1e-9)) + hi = int(math.floor(self.max / self.step + 1e-9)) + for i in xrange(lo, hi + 1): + value = i * self.step + if self.min == 0 and i == 0: + continue + yield self.mapAxisValue(self.fromAxis(value)), self.labeler(self.fromAxis(value)) + + +#--------------------------------------------------- +# Graph +#--------------------------------------------------- +def makeNamedTuples(results, typeName = 'Point', labels = 'labels', points = 'points'): + namedTupleType = collections.namedtuple(typeName, results[labels]) + numLabels = len(results[labels]) + return [namedTupleType(*p[:numLabels]) for p in results[points]] + +class Curve: + def __init__(self, name, points, color): + self.name = name + self.points = points + self.color = color + +class Graph: + """ Renders a graph. """ + + def __init__(self, xAttribs, yAttribs): + self.xAttribs = xAttribs + self.yAttribs = yAttribs + self.curves = [] + self.xMin = None + self.xMax = None + + def addCurve(self, curve): + self.curves.append(curve) + xMin = min(x for x, y in curve.points) + if self.xMin is None or xMin < self.xMin: + self.xMin = xMin + xMax = max(x for x, y in curve.points) + if self.xMax is None or xMax > self.xMax: + self.xMax = xMax + + def renderTo(self, fileName): + xAttribs = self.xAttribs + yAttribs = self.yAttribs + xAttribs.setMinMax(self.xMin, self.xMax) + + # Create the image surface and cairo context + surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, 330 + int(xAttribs.size + 0.5), 55 + int(yAttribs.size + 0.5)) + cr = cairo.Context(surface) + cr.set_source_rgb(1, 1, 1) + cr.paint() + cr.set_miter_limit(1.414) + cr.translate(80, 8 + yAttribs.size) + + # Draw axes + labelFont = createScaledFont('Arial', 13) + with Saved(cr): + cr.set_line_width(1) + cr.set_source_rgb(.4, .4, .4) + + # Horizontal axis + cr.move_to(0, -0.5) + cr.rel_line_to(xAttribs.size + 1, 0) + cr.stroke() + for pos, label in xAttribs.iterLabels(): # Labels + x = math.floor(pos + 0.5) + with Saved(cr): + cr.translate(x, 9) + fillAlignedText(cr, 0, 6, labelFont, label, 0.5) + + # Vertical axis + cr.set_source_rgb(*colorTuple('f0f0f0')) + for pos, label in yAttribs.iterLabels(): # Background lines + if label == '0': + continue + y = -math.floor(pos + 0.5) - 0.5 + cr.move_to(1, y) + cr.rel_line_to(xAttribs.size + 1, 0) + cr.stroke() + cr.set_source_rgb(.4, .4, .4) + cr.move_to(0.5, 0) + cr.rel_line_to(0, -yAttribs.size - 0.5) + if False: + for pos, label in yAttribs.iterLabels(): # Tick marks + if label == '0': + continue + y = -math.floor(pos + 0.5) - 0.5 + cr.move_to(1, y) + cr.rel_line_to(-4, 0) + cr.stroke() + for pos, label in yAttribs.iterLabels(): # Labels + if label == '0': + continue + fillAlignedText(cr, -4, -pos + 4, labelFont, label, 1) + + """ + with Saved(cr): + x = xAttribs.size - 70.5 + 80 + y = -234.5 + cr.rectangle(x, y, 120, 82) + cr.set_source_rgb(*colorTuple('ffffff')) + cr.fill() + cr.set_source_rgb(*colorTuple('f0f0f0')) + cr.rectangle(x, y, 120, 82) + cr.set_line_width(1) + cr.stroke() + """ + + # Draw curves + for cn, curve in enumerate(self.curves): + points = curve.points + color = curve.color + width = 1.75 + #if color == colorTuple('ff4040'): + #width = 2 + with Saved(cr): + cr.set_line_width(width) + cr.set_source_rgba(*color) + #if color == colorTuple('9090b0'): + # cr.set_dash([9, 2]) + with Saved(cr): + cr.rectangle(0, 5, xAttribs.size, -yAttribs.size - 15) + cr.clip() + x, y = points[0] + cr.move_to(xAttribs.mapAxisValue(x), -yAttribs.mapAxisValue(y)) + for x, y in points[1:]: + cr.line_to(xAttribs.mapAxisValue(x) + 0.5, -yAttribs.mapAxisValue(y) - 0.5) + cr.stroke() + for x, y in points: + cr.rectangle(xAttribs.mapAxisValue(x) - 2.5, -yAttribs.mapAxisValue(y) - 2.5, 5, 5) + cr.fill() + + x = xAttribs.size + 40 + y = -120 + (5 - cn) * 14 + cr.move_to(x - 4.5, y - 4.5) + cr.rel_line_to(-21, 0) + cr.stroke() + + # Label + weight = cairo.FONT_WEIGHT_NORMAL + if color == colorTuple('ff4040'): + weight = cairo.FONT_WEIGHT_BOLD + labelFont = createScaledFont('Arial', 13, weight=weight) + label = curve.name + #x, y = points[-1] + #fillAlignedText(cr, xAttribs.mapAxisValue(x) + 3, -yAttribs.mapAxisValue(y) + 4, labelFont, label, 0) + x = xAttribs.size + 40 + y = -120 + (5 - cn) * 14 + fillAlignedText(cr, x, y, labelFont, label, 0) + + # Draw axis names + cr.set_source_rgb(0, 0, 0) + axisFont = createScaledFont('Helvetica', 16, weight=cairo.FONT_WEIGHT_BOLD) + with Saved(cr): + cr.translate(-66, -yAttribs.size / 2.0) + cr.rotate(-math.pi / 2) + fillAlignedText(cr, 0, 0, axisFont, 'Map Operations / Sec', 0.5) + with Saved(cr): + axisFont2 = createScaledFont('Helvetica', 13) + cr.translate(-50, -yAttribs.size / 2.0) + cr.rotate(-math.pi / 2) + cr.set_source_rgba(*colorTuple('808080')) + fillAlignedText(cr, 0, 0, axisFont2, '(Total Across All Threads)', 0.5) + fillAlignedText(cr, xAttribs.size / 2.0, 42, axisFont, 'Threads', 0.5) + + # Save PNG file + surface.write_to_png(fileName) + + +#--------------------------------------------------- +# main +#--------------------------------------------------- +def formatTime(x): + if x < 0.9e-6: + return '%d ns' % (x * 1e9 + 0.5) + elif x < 0.9e-3: + return '%d us' % (x * 1e6 + 0.5) # FIXME: Micro symbol + if x < 0.9: + return '%d ms' % (x * 1e3 + 0.5) + else: + return '%d s' % (x + 0.5) + +graph = Graph(AxisAttribs(250, 1, 6, 1), + AxisAttribs(240, 0, 150, 10, False, lambda x: '%dM' % x if x % 50 == 0 else '')) + +for suffix, color in ALL_MAPS: + resultsPath = 'build-%s/results.txt' % suffix + if os.path.exists(resultsPath): + with open(resultsPath, 'r') as f: + results = eval(f.read()) + dataPoints = makeNamedTuples(results) + def makeGraphPoint(pt): + mapOpsPerSec = pt.mapOpsDone / pt.totalTime + return (pt.numThreads, mapOpsPerSec * pt.numThreads / 1000000) + graphPoints = [makeGraphPoint(pt) for pt in dataPoints] + graph.addCurve(Curve(results['mapType'], graphPoints, color)) + +graph.renderTo('out.png') diff --git a/samples/MapScalabilityTests/TestAllMaps.py b/samples/MapScalabilityTests/TestAllMaps.py new file mode 100644 index 0000000..f60f167 --- /dev/null +++ b/samples/MapScalabilityTests/TestAllMaps.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +import os +import subprocess +import sys + +# Default CMake command is just 'cmake' but you can override it by setting +# the CMAKE environment variable: +CMAKE = os.getenv('CMAKE', 'cmake') + +ALL_MAPS = [ + ('michael', 'junction/extra/impl/MapAdapter_CDS_Michael.h', ['-DJUNCTION_WITH_CDS=1', '-DTURF_WITH_EXCEPTIONS=1'], ['-i10000', '-c200']), + ('linear', 'junction/extra/impl/MapAdapter_Linear.h', [], ['-i10000', '-c200']), + ('leapfrog', 'junction/extra/impl/MapAdapter_LeapFrog.h', [], ['-i10000', '-c200']), + ('grampa', 'junction/extra/impl/MapAdapter_Grampa.h', [], ['-i10000', '-c200']), + ('stdmap', 'junction/extra/impl/MapAdapter_StdMap.h', [], ['-i10000', '-c10']), + ('folly', 'junction/extra/impl/MapAdapter_Folly.h', ['-DJUNCTION_WITH_FOLLY=1', '-DTURF_WITH_EXCEPTIONS=1'], ['-i2000', '-c1']), + ('nbds', 'junction/extra/impl/MapAdapter_NBDS.h', ['-DJUNCTION_WITH_NBDS=1'], ['-i10000', '-c200']), + ('tbb', 'junction/extra/impl/MapAdapter_TBB.h', ['-DJUNCTION_WITH_TBB=1'], ['-i10000', '-c200']), + ('tervel', 'junction/extra/impl/MapAdapter_Tervel.h', ['-DJUNCTION_WITH_TERVEL=1'], ['-i1000', '-c20']), +] + +# Scan arguments for path to CMakeLists.txt and args to pass through. +passThroughArgs = [] +pathArgs = [] +for arg in sys.argv[1:]: + if arg.startswith('-'): + passThroughArgs.append(arg) + else: + pathArgs.append(arg) +if len(pathArgs) != 1: + sys.stderr.write('You must provide exactly one path argument.\n') + exit(1) + +listFilePath = os.path.abspath(pathArgs[0]) +for suffix, include, cmakeOpts, runtimeOpts in ALL_MAPS: + success = True + subdir = 'build-%s' % suffix + if not os.path.exists(subdir): + os.mkdir(subdir) + os.chdir(subdir) + print('Configuring in %s...' % subdir) + with open('junction_userconfig.h.in', 'w') as f: + f.write('#define JUNCTION_IMPL_MAPADAPTER_PATH "%s"\n' % include) + userConfigCMakePath = os.path.abspath('junction_userconfig.h.in') + if os.sep != '/': + userConfigCMakePath = userConfigCMakePath.replace(os.sep, '/') + if subprocess.call([CMAKE, listFilePath, '-DCMAKE_BUILD_TYPE=RelWithDebInfo', '-DCMAKE_INSTALL_PREFIX=TestAllMapsInstallFolder', + '-DJUNCTION_USERCONFIG=%s' % userConfigCMakePath] + passThroughArgs + cmakeOpts) == 0: + subprocess.check_call([CMAKE, '--build', '.', '--target', 'install', '--config', 'RelWithDebInfo']) + print('Running in %s...' % subdir) + results = subprocess.check_output([os.path.join('TestAllMapsInstallFolder', 'bin', 'MapScalabilityTests')] + runtimeOpts) + with open('results.txt', 'wb') as f: + f.write(results) + os.chdir('..') -- 2.34.1