2 * Copyright (c) 2015, Facebook, Inc.
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree. An additional grant
7 * of patent rights can be found in the PATENTS file in the same directory.
12 #include <folly/wangle/ssl/SSLCacheProvider.h>
13 #include <folly/wangle/ssl/SSLStats.h>
15 #include <folly/EvictingCacheMap.h>
17 #include <folly/io/async/AsyncSSLSocket.h>
24 * Basic SSL session cache map: Maps session id -> session
26 typedef folly::EvictingCacheMap<std::string, SSL_SESSION*> SSLSessionCacheMap;
29 * Holds an SSLSessionCacheMap and associated lock
31 class LocalSSLSessionCache: private boost::noncopyable {
33 LocalSSLSessionCache(uint32_t maxCacheSize, uint32_t cacheCullSize);
35 ~LocalSSLSessionCache() {
36 std::lock_guard<std::mutex> g(lock);
37 // EvictingCacheMap dtor doesn't free values
41 SSLSessionCacheMap sessionCache;
43 uint32_t removedSessions_{0};
47 void pruneSessionCallback(const std::string& sessionId,
48 SSL_SESSION* session);
52 * A sharded LRU for SSL sessions. The sharding is inteneded to reduce
53 * contention for the LRU locks. Assuming uniform distribution, two workers
54 * will contend for the same lock with probability 1 / n_buckets^2.
56 class ShardedLocalSSLSessionCache : private boost::noncopyable {
58 ShardedLocalSSLSessionCache(uint32_t n_buckets, uint32_t maxCacheSize,
59 uint32_t cacheCullSize) {
61 maxCacheSize = (uint32_t)(((double)maxCacheSize) / n_buckets);
62 cacheCullSize = (uint32_t)(((double)cacheCullSize) / n_buckets);
63 if (maxCacheSize == 0) {
66 if (cacheCullSize == 0) {
69 for (uint32_t i = 0; i < n_buckets; i++) {
71 std::unique_ptr<LocalSSLSessionCache>(
72 new LocalSSLSessionCache(maxCacheSize, cacheCullSize)));
76 SSL_SESSION* lookupSession(const std::string& sessionId) {
77 size_t bucket = hash(sessionId);
78 SSL_SESSION* session = nullptr;
79 std::lock_guard<std::mutex> g(caches_[bucket]->lock);
81 auto itr = caches_[bucket]->sessionCache.find(sessionId);
82 if (itr != caches_[bucket]->sessionCache.end()) {
83 session = itr->second;
87 CRYPTO_add(&session->references, 1, CRYPTO_LOCK_SSL_SESSION);
92 void storeSession(const std::string& sessionId, SSL_SESSION* session,
94 size_t bucket = hash(sessionId);
95 SSL_SESSION* oldSession = nullptr;
96 std::lock_guard<std::mutex> g(caches_[bucket]->lock);
98 auto itr = caches_[bucket]->sessionCache.find(sessionId);
99 if (itr != caches_[bucket]->sessionCache.end()) {
100 oldSession = itr->second;
104 // LRUCacheMap doesn't free on overwrite, so 2x the work for us
105 // This can happen in race conditions
106 SSL_SESSION_free(oldSession);
108 caches_[bucket]->removedSessions_ = 0;
109 caches_[bucket]->sessionCache.set(sessionId, session, true);
111 stats->recordSSLSessionFree(caches_[bucket]->removedSessions_);
115 void removeSession(const std::string& sessionId) {
116 size_t bucket = hash(sessionId);
117 std::lock_guard<std::mutex> g(caches_[bucket]->lock);
118 caches_[bucket]->sessionCache.erase(sessionId);
123 /* SSL session IDs are 32 bytes of random data, hash based on first 16 bits */
124 size_t hash(const std::string& key) {
125 CHECK(key.length() >= 2);
126 return (key[0] << 8 | key[1]) % caches_.size();
129 std::vector< std::unique_ptr<LocalSSLSessionCache> > caches_;
132 /* A socket/DestructorGuard pair */
133 typedef std::pair<AsyncSSLSocket *,
134 std::unique_ptr<DelayedDestruction::DestructorGuard>>
138 * PendingLookup structure
140 * Keeps track of clients waiting for an SSL session to be retrieved from
141 * the external cache provider.
143 struct PendingLookup {
144 bool request_in_progress;
145 SSL_SESSION* session;
146 std::list<AttachedLookup> waiters;
149 request_in_progress = true;
154 /* Maps SSL session id to a PendingLookup structure */
155 typedef std::map<std::string, PendingLookup> PendingLookupMap;
158 * SSLSessionCacheManager handles all stateful session caching. There is an
159 * instance of this object per SSL VIP per thread, with a 1:1 correlation with
160 * SSL_CTX. The cache can work locally or in concert with an external cache
161 * to share sessions across instances.
163 * There is a single in memory session cache shared by all VIPs. The cache is
164 * split into N buckets (currently 16) with a separate lock per bucket. The
165 * VIP ID is hashed and stored as part of the session to handle the
166 * (very unlikely) case of session ID collision.
168 * When a new SSL session is created, it is added to the LRU cache and
169 * sent to the external cache to be stored. The external cache
170 * expiration is equal to the SSL session's expiration.
172 * When a resume request is received, SSLSessionCacheManager first looks in the
173 * local LRU cache for the VIP. If there is a miss there, an asynchronous
174 * request for this session is dispatched to the external cache. When the
175 * external cache query returns, the LRU cache is updated if the session was
176 * found, and the SSL_accept call is resumed.
178 * If additional resume requests for the same session ID arrive in the same
179 * thread while the request is pending, the 2nd - Nth callers attach to the
180 * original external cache requests and are resumed when it comes back. No
181 * attempt is made to coalesce external cache requests for the same session
182 * ID in different worker threads. Previous work did this, but the
183 * complexity was deemed to outweigh the potential savings.
186 class SSLSessionCacheManager : private boost::noncopyable {
189 * Constructor. SSL session related callbacks will be set on the underlying
190 * SSL_CTX. vipId is assumed to a unique string identifying the VIP and must
191 * be the same on all servers that wish to share sessions via the same
194 SSLSessionCacheManager(
195 uint32_t maxCacheSize,
196 uint32_t cacheCullSize,
198 const folly::SocketAddress& sockaddr,
199 const std::string& context,
200 EventBase* eventBase,
202 const std::shared_ptr<SSLCacheProvider>& externalCache);
204 virtual ~SSLSessionCacheManager();
207 * Call this on shutdown to release the global instance of the
208 * ShardedLocalSSLSessionCache.
210 static void shutdown();
213 * Callback for ExternalCache to call when an async get succeeds
214 * @param context The context that was passed to the async get request
215 * @param value Serialized session
217 void onGetSuccess(SSLCacheProvider::CacheContext* context,
218 const std::string& value);
221 * Callback for ExternalCache to call when an async get fails, either
222 * because the requested session is not in the external cache or because
224 * @param context The context that was passed to the async get request
226 void onGetFailure(SSLCacheProvider::CacheContext* context);
231 std::shared_ptr<ShardedLocalSSLSessionCache> localCache_;
232 PendingLookupMap pendingLookups_;
233 SSLStats* stats_{nullptr};
234 std::shared_ptr<SSLCacheProvider> externalCache_;
237 * Invoked by openssl when a new SSL session is created
239 int newSession(SSL* ssl, SSL_SESSION* session);
242 * Invoked by openssl when an SSL session is ejected from its internal cache.
243 * This can't be invoked in the current implementation because SSL's internal
244 * caching is disabled.
246 void removeSession(SSL_CTX* ctx, SSL_SESSION* session);
249 * Invoked by openssl when a client requests a stateful session resumption.
250 * Triggers a lookup in our local cache and potentially an asynchronous
251 * request to an external cache.
253 SSL_SESSION* getSession(SSL* ssl, unsigned char* session_id,
254 int id_len, int* copyflag);
257 * Store a new session record in the external cache
259 bool storeCacheRecord(const std::string& sessionId, SSL_SESSION* session);
262 * Lookup a session in the external cache for the specified SSL socket.
264 bool lookupCacheRecord(const std::string& sessionId,
265 AsyncSSLSocket* sslSock);
268 * Restart all clients waiting for the answer to an external cache query
270 void restartSSLAccept(const SSLCacheProvider::CacheContext* cacheCtx);
273 * Get or create the LRU cache for the given VIP ID
275 static std::shared_ptr<ShardedLocalSSLSessionCache> getLocalCache(
276 uint32_t maxCacheSize, uint32_t cacheCullSize);
279 * static functions registered as callbacks to openssl via
280 * SSL_CTX_sess_set_new/get/remove_cb
282 static int newSessionCallback(SSL* ssl, SSL_SESSION* session);
283 static void removeSessionCallback(SSL_CTX* ctx, SSL_SESSION* session);
284 static SSL_SESSION* getSessionCallback(SSL* ssl, unsigned char* session_id,
285 int id_len, int* copyflag);
287 static int32_t sExDataIndex_;
288 static std::shared_ptr<ShardedLocalSSLSessionCache> sCache_;
289 static std::mutex sCacheLock_;