openssl: reduce memory consumption

SSL records may be as large as 16K, but are typically < 2K.  In
addition, a historic bug in Windows allowed records to be as large
32K.  OpenSSL statically allocates read and write buffers (34K and
18K respectively) used for processing records.

With this patch, OpenSSL statically allocates 4K + 4K buffers, with
the option of dynamically growing buffers to 34K + 4K, which is a
saving of 44K per connection for the typical case.

This patch is taken from the Android Open Source Project.

deps/openssl/openssl/ssl/d1_pkt.c

@@ -614,6 +614,24 @@ again:
 			goto again;
 			}
 
+		/* If we receive a valid record larger than the current buffer size,
+		 * allocate some memory for it.
+		 */
+		if (rr->length > s->s3->rbuf.len - DTLS1_RT_HEADER_LENGTH)
+			{
+			unsigned char *pp;
+			unsigned int newlen = rr->length + DTLS1_RT_HEADER_LENGTH;
+			if ((pp=OPENSSL_realloc(s->s3->rbuf.buf, newlen))==NULL)
+				{
+				SSLerr(SSL_F_DTLS1_GET_RECORD,ERR_R_MALLOC_FAILURE);
+				return(-1);
+				}
+			p = pp + (p - s->s3->rbuf.buf);
+			s->s3->rbuf.buf=pp;
+			s->s3->rbuf.len=newlen;
+			s->packet= &(s->s3->rbuf.buf[0]);
+			}
+
 		/* now s->rstate == SSL_ST_READ_BODY */
 		}
 
@@ -1452,6 +1470,7 @@ int do_dtls1_write(SSL *s, int type, const unsigned char *buf, unsigned int len,
 	SSL3_BUFFER *wb;
 	SSL_SESSION *sess;
 	int bs;
+	unsigned int len_with_overhead = len + SSL3_RT_DEFAULT_WRITE_OVERHEAD;
 
 	/* first check if there is a SSL3_BUFFER still being written
 	 * out.  This will happen with non blocking IO */
@@ -1461,6 +1480,16 @@ int do_dtls1_write(SSL *s, int type, const unsigned char *buf, unsigned int len,
 		return(ssl3_write_pending(s,type,buf,len));
 		}
 
+	if (s->s3->wbuf.len < len_with_overhead)
+		{
+		if ((p=OPENSSL_realloc(s->s3->wbuf.buf, len_with_overhead)) == NULL) {
+			SSLerr(SSL_F_DO_DTLS1_WRITE,ERR_R_MALLOC_FAILURE);
+			goto err;
+		}
+		s->s3->wbuf.buf = p;
+		s->s3->wbuf.len = len_with_overhead;
+		}
+
 	/* If we have an alert to send, lets send it */
 	if (s->s3->alert_dispatch)
 		{

deps/openssl/openssl/ssl/s23_srvr.c

@@ -444,8 +444,13 @@ int ssl23_get_client_hello(SSL *s)
 		v[0] = p[3]; /* == SSL3_VERSION_MAJOR */
 		v[1] = p[4];
 
+/* The SSL2 protocol allows n to be larger, just pick
+ * a reasonable buffer size. */
+#if SSL3_RT_DEFAULT_PACKET_SIZE < 1024*4 - SSL3_RT_DEFAULT_WRITE_OVERHEAD
+#error "SSL3_RT_DEFAULT_PACKET_SIZE is too small."
+#endif
 		n=((p[0]&0x7f)<<8)|p[1];
-		if (n > (1024*4))
+		if (n > SSL3_RT_DEFAULT_PACKET_SIZE - 2)
 			{
 			SSLerr(SSL_F_SSL23_GET_CLIENT_HELLO,SSL_R_RECORD_TOO_LARGE);
 			goto err;

deps/openssl/openssl/ssl/s3_both.c

@@ -745,13 +745,20 @@ int ssl3_setup_read_buffer(SSL *s)
 
 	if (s->s3->rbuf.buf == NULL)
 		{
-		len = SSL3_RT_MAX_PLAIN_LENGTH
-			+ SSL3_RT_MAX_ENCRYPTED_OVERHEAD
-			+ headerlen + align;
-		if (s->options & SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER)
+		if (SSL_get_mode(s) & SSL_MODE_SMALL_BUFFERS)
+			{
+			len = SSL3_RT_DEFAULT_PACKET_SIZE;
+			}
+  		else
 			{
-			s->s3->init_extra = 1;
-			len += SSL3_RT_MAX_EXTRA;
+			len = SSL3_RT_MAX_PLAIN_LENGTH
+				+ SSL3_RT_MAX_ENCRYPTED_OVERHEAD
+				+ headerlen + align;
+			if (s->options & SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER)
+				{
+				s->s3->init_extra = 1;
+				len += SSL3_RT_MAX_EXTRA;
+				}
 			}
 #ifndef OPENSSL_NO_COMP
 		if (!(s->options & SSL_OP_NO_COMPRESSION))
@@ -787,7 +794,15 @@ int ssl3_setup_write_buffer(SSL *s)
 
 	if (s->s3->wbuf.buf == NULL)
 		{
-		len = s->max_send_fragment
+		if (SSL_get_mode(s) & SSL_MODE_SMALL_BUFFERS)
+			{
+			len = SSL3_RT_DEFAULT_PACKET_SIZE;
+			}
+  		else
+			{
+			len = s->max_send_fragment;
+			}
+		len += 0
 			+ SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD
 			+ headerlen + align;
 #ifndef OPENSSL_NO_COMP
@@ -797,7 +812,6 @@ int ssl3_setup_write_buffer(SSL *s)
 		if (!(s->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS))
 			len += headerlen + align
 				+ SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD;
-
 		if ((p=freelist_extract(s->ctx, 0, len)) == NULL)
 			goto err;
 		s->s3->wbuf.buf = p;
@@ -840,4 +854,3 @@ int ssl3_release_read_buffer(SSL *s)
 		}
 	return 1;
 	}
-

deps/openssl/openssl/ssl/s3_pkt.c

@@ -295,6 +295,11 @@ static int ssl3_get_record(SSL *s)
 	size_t extra;
 	int decryption_failed_or_bad_record_mac = 0;
 	unsigned char *mac = NULL;
+#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0
+	long align=SSL3_ALIGN_PAYLOAD;
+#else
+	long align=0;
+#endif
 
 	rr= &(s->s3->rrec);
 	sess=s->session;
@@ -303,7 +308,8 @@ static int ssl3_get_record(SSL *s)
 		extra=SSL3_RT_MAX_EXTRA;
 	else
 		extra=0;
-	if (extra && !s->s3->init_extra)
+	if (!(SSL_get_mode(s) & SSL_MODE_SMALL_BUFFERS) &&
+		extra && !s->s3->init_extra)
 		{
 		/* An application error: SLS_OP_MICROSOFT_BIG_SSLV3_BUFFER
 		 * set after ssl3_setup_buffers() was done */
@@ -352,6 +358,21 @@ fprintf(stderr, "Record type=%d, Length=%d\n", rr->type, rr->length);
 			goto err;
 			}
 
+		/* If we receive a valid record larger than the current buffer size,
+		 * allocate some memory for it.
+		 */
+		if (rr->length > s->s3->rbuf.len - SSL3_RT_HEADER_LENGTH - align)
+			{
+			if ((p=OPENSSL_realloc(s->s3->rbuf.buf, rr->length + SSL3_RT_HEADER_LENGTH + align))==NULL)
+				{
+				SSLerr(SSL_F_SSL3_GET_RECORD,ERR_R_MALLOC_FAILURE);
+				goto err;
+				}
+			s->s3->rbuf.buf=p;
+			s->s3->rbuf.len=rr->length + SSL3_RT_HEADER_LENGTH + align;
+			s->packet= &(s->s3->rbuf.buf[0]);
+			}
+
 		if (rr->length > s->s3->rbuf.len - SSL3_RT_HEADER_LENGTH)
 			{
 			al=SSL_AD_RECORD_OVERFLOW;
@@ -578,6 +599,7 @@ int ssl3_write_bytes(SSL *s, int type, const void *buf_, int len)
 	const unsigned char *buf=buf_;
 	unsigned int tot,n,nw;
 	int i;
+	unsigned int max_plain_length;
 
 	s->rwstate=SSL_NOTHING;
 	tot=s->s3->wnum;
@@ -597,8 +619,13 @@ int ssl3_write_bytes(SSL *s, int type, const void *buf_, int len)
 	n=(len-tot);
 	for (;;)
 		{
-		if (n > s->max_send_fragment)
-			nw=s->max_send_fragment;
+		if (type == SSL3_RT_APPLICATION_DATA && (SSL_get_mode(s) & SSL_MODE_SMALL_BUFFERS))
+			max_plain_length = SSL3_RT_DEFAULT_PLAIN_LENGTH;
+		else
+			max_plain_length = s->max_send_fragment;
+
+		if (n > max_plain_length)
+			nw = max_plain_length;
 		else
 			nw=n;
 
@@ -707,6 +734,18 @@ static int do_ssl3_write(SSL *s, int type, const unsigned char *buf,
 		s->s3->empty_fragment_done = 1;
 		}
 
+	/* resize if necessary to hold the data. */
+	if (len + SSL3_RT_DEFAULT_WRITE_OVERHEAD > wb->len)
+		{
+		if ((p=OPENSSL_realloc(wb->buf, len + SSL3_RT_DEFAULT_WRITE_OVERHEAD))==NULL)
+			{
+			SSLerr(SSL_F_DO_SSL3_WRITE,ERR_R_MALLOC_FAILURE);
+			goto err;
+			}
+		wb->buf = p;
+		wb->len = len + SSL3_RT_DEFAULT_WRITE_OVERHEAD;
+		}
+
 	if (create_empty_fragment)
 		{
 #if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0

deps/openssl/openssl/ssl/ssl.h

@@ -638,6 +638,9 @@ struct ssl_session_st
  * TLS only.)  "Released" buffers are put onto a free-list in the context
  * or just freed (depending on the context's setting for freelist_max_len). */
 #define SSL_MODE_RELEASE_BUFFERS 0x00000010L
+/* Use small read and write buffers: (a) lazy allocate read buffers for
+ * large incoming records, and (b) limit the size of outgoing records. */
+#define SSL_MODE_SMALL_BUFFERS 0x00000020L
 
 /* Note: SSL[_CTX]_set_{options,mode} use |= op on the previous value,
  * they cannot be used to clear bits. */

deps/openssl/openssl/ssl/ssl3.h

@@ -280,6 +280,9 @@ extern "C" {
 
 #define SSL3_RT_MAX_EXTRA			(16384)
 
+/* Default buffer length used for writen records.  Thus a generated record
+ * will contain plaintext no larger than this value. */
+#define SSL3_RT_DEFAULT_PLAIN_LENGTH	2048
 /* Maximum plaintext length: defined by SSL/TLS standards */
 #define SSL3_RT_MAX_PLAIN_LENGTH		16384
 /* Maximum compression overhead: defined by SSL/TLS standards */
@@ -311,6 +314,13 @@ extern "C" {
 #define SSL3_RT_MAX_PACKET_SIZE		\
 		(SSL3_RT_MAX_ENCRYPTED_LENGTH+SSL3_RT_HEADER_LENGTH)
 
+/* Extra space for empty fragment, headers, MAC, and padding. */
+#define SSL3_RT_DEFAULT_WRITE_OVERHEAD  256
+#define SSL3_RT_DEFAULT_PACKET_SIZE     4096 - SSL3_RT_DEFAULT_WRITE_OVERHEAD
+#if SSL3_RT_DEFAULT_PLAIN_LENGTH + SSL3_RT_DEFAULT_WRITE_OVERHEAD > SSL3_RT_DEFAULT_PACKET_SIZE
+#error "Insufficient space allocated for write buffers."
+#endif
+
 #define SSL3_MD_CLIENT_FINISHED_CONST	"\x43\x4C\x4E\x54"
 #define SSL3_MD_SERVER_FINISHED_CONST	"\x53\x52\x56\x52"
 
@@ -675,4 +685,3 @@ typedef struct ssl3_state_st
 }
 #endif
 #endif
-

deps/openssl/openssl/ssl/ssltest.c

@@ -369,6 +369,8 @@ static void sv_usage(void)
 	               "                 (default is sect163r2).\n");
 #endif
 	fprintf(stderr," -test_cipherlist - verifies the order of the ssl cipher lists\n");
+	fprintf(stderr," -c_small_records - enable client side use of small SSL record buffers\n");
+	fprintf(stderr," -s_small_records - enable server side use of small SSL record buffers\n");
 	}
 
 static void print_details(SSL *c_ssl, const char *prefix)
@@ -497,6 +499,9 @@ int opaque_prf_input_cb(SSL *ssl, void *peerinput, size_t len, void *arg_)
 	return arg->ret;
 	}
 #endif
+	int ssl_mode = 0;
+	int c_small_records=0;
+	int s_small_records=0;
 
 int main(int argc, char *argv[])
 	{
@@ -765,6 +770,14 @@ int main(int argc, char *argv[])
 			{
 			test_cipherlist = 1;
 			}
+		else if (strcmp(*argv, "-c_small_records") == 0)
+			{
+			c_small_records = 1;
+			}
+		else if (strcmp(*argv, "-s_small_records") == 0)
+			{
+			s_small_records = 1;
+			}
 		else
 			{
 			fprintf(stderr,"unknown option %s\n",*argv);
@@ -901,6 +914,21 @@ bad:
 		SSL_CTX_set_cipher_list(s_ctx,cipher);
 		}
 
+	ssl_mode = 0;
+	if (c_small_records)
+		{
+		ssl_mode = SSL_CTX_get_mode(c_ctx);
+		ssl_mode |= SSL_MODE_SMALL_BUFFERS;
+		SSL_CTX_set_mode(c_ctx, ssl_mode);
+		}
+	ssl_mode = 0;
+	if (s_small_records)
+		{
+		ssl_mode = SSL_CTX_get_mode(s_ctx);
+		ssl_mode |= SSL_MODE_SMALL_BUFFERS;
+		SSL_CTX_set_mode(s_ctx, ssl_mode);
+		}
+
 #ifndef OPENSSL_NO_DH
 	if (!no_dhe)
 		{

deps/openssl/openssl/test/testssl

@@ -70,6 +70,15 @@ $ssltest -client_auth $CA $extra || exit 1
 echo test sslv2/sslv3 with both client and server authentication
 $ssltest -server_auth -client_auth $CA $extra || exit 1
 
+echo test sslv2/sslv3 with both client and server authentication and small client buffers
+$ssltest -server_auth -client_auth -c_small_records $CA $extra || exit 1
+
+echo test sslv2/sslv3 with both client and server authentication and small server buffers
+$ssltest -server_auth -client_auth -s_small_records $CA $extra || exit 1
+
+echo test sslv2/sslv3 with both client and server authentication and small client and server buffers
+$ssltest -server_auth -client_auth -c_small_records -s_small_records $CA $extra || exit 1
+
 echo test sslv2 via BIO pair
 $ssltest -bio_pair -ssl2 $extra || exit 1