From nobody@FreeBSD.org Wed Sep 17 08:09:56 2008 Return-Path: Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 772E21065678 for ; Wed, 17 Sep 2008 08:09:56 +0000 (UTC) (envelope-from nobody@FreeBSD.org) Received: from www.freebsd.org (www.freebsd.org [IPv6:2001:4f8:fff6::21]) by mx1.freebsd.org (Postfix) with ESMTP id 669528FC1E for ; Wed, 17 Sep 2008 08:09:56 +0000 (UTC) (envelope-from nobody@FreeBSD.org) Received: from www.freebsd.org (localhost [127.0.0.1]) by www.freebsd.org (8.14.2/8.14.2) with ESMTP id m8H89uc4050007 for ; Wed, 17 Sep 2008 08:09:56 GMT (envelope-from nobody@www.freebsd.org) Received: (from nobody@localhost) by www.freebsd.org (8.14.2/8.14.1/Submit) id m8H89tRQ050006; Wed, 17 Sep 2008 08:09:55 GMT (envelope-from nobody) Message-Id: <200809170809.m8H89tRQ050006@www.freebsd.org> Date: Wed, 17 Sep 2008 08:09:55 GMT From: johnpupu To: freebsd-gnats-submit@FreeBSD.org Subject: lighttpd mod_deflate patch for freebsd X-Send-Pr-Version: www-3.1 X-GNATS-Notify: >Number: 127435 >Category: ports >Synopsis: [patch] www/lighttpd mod_deflate patch for freebsd >Confidential: no >Severity: non-critical >Priority: low >Responsible: mnag >State: closed >Quarter: >Keywords: >Date-Required: >Class: sw-bug >Submitter-Id: current-users >Arrival-Date: Wed Sep 17 08:20:03 UTC 2008 >Closed-Date: Sun Jun 28 22:17:56 UTC 2009 >Last-Modified: Sun Jun 28 22:17:56 UTC 2009 >Originator: johnpupu >Release: 7.0 >Organization: mml >Environment: FreeBSD mail.mml.com.tw 7.0-RELEASE-p4 FreeBSD 7.0-RELEASE-p4 #1: Thu Sep 4 12:18:31 CST 2008 root@mail.mml.com.tw:/usr/obj/usr/src/sys/MAIL i386 >Description: lighttpd mod_deflate patch for freebsd >How-To-Repeat: >Fix: *** ../lighttpd-1.4.19/configure.in 2008-09-17 14:34:08.000000000 +0800 --- configure.in 2008-09-17 14:42:22.000000000 +0800 *************** *** 556,566 **** lighttpd.spec distribute.sh cygwin/Makefile cygwin/lighttpd.README openwrt/Makefile openwrt/control openwrt/lighttpd.mk]) AC_OUTPUT ! do_build="mod_cgi mod_fastcgi mod_extforward mod_proxy mod_evhost mod_simple_vhost mod_access mod_alias mod_setenv mod_usertrack mod_auth mod_status mod_accesslog mod_rrdtool mod_secdownload mod_expire mod_compress mod_dirlisting mod_indexfiles mod_userdir mod_webdav mod_staticfile mod_scgi mod_flv_streaming" plugins="mod_rewrite mod_redirect mod_ssi mod_trigger_b4_dl" features="regex-conditionals" if test ! "x$PCRE_LIB" = x; then do_build="$do_build $plugins" --- 556,566 ---- lighttpd.spec distribute.sh cygwin/Makefile cygwin/lighttpd.README openwrt/Makefile openwrt/control openwrt/lighttpd.mk]) AC_OUTPUT ! do_build="mod_cgi mod_fastcgi mod_extforward mod_proxy mod_evhost mod_simple_vhost mod_access mod_alias mod_setenv mod_usertrack mod_auth mod_status mod_accesslog mod_rrdtool mod_secdownload mod_expire mod_compress mod_dirlisting mod_indexfiles mod_userdir mod_webdav mod_staticfile mod_scgi mod_flv_streaming mod_deflate" plugins="mod_rewrite mod_redirect mod_ssi mod_trigger_b4_dl" features="regex-conditionals" if test ! "x$PCRE_LIB" = x; then do_build="$do_build $plugins" *** ../lighttpd-1.4.19/src/base.h 2008-01-18 17:07:54.000000000 +0800 --- src/base.h 2008-09-17 14:42:22.000000000 +0800 *************** *** 147,156 **** --- 147,157 ---- buffer *orig_uri; http_method_t http_method; http_version_t http_version; + int true_http_10_client; buffer *request_line; /* strings to the header */ buffer *http_host; /* not alloced */ *************** *** 358,369 **** int keep_alive; /* only request.c can enable it, all other just disable */ int file_started; int file_finished; ! chunkqueue *write_queue; /* a large queue for low-level write ( HTTP response ) [ file, mem ] */ chunkqueue *read_queue; /* a small queue for low-level read ( HTTP request ) [ mem ] */ chunkqueue *request_content_queue; /* takes request-content into tempfile if necessary [ tempfile, mem ]*/ int traffic_limit_reached; --- 359,372 ---- int keep_alive; /* only request.c can enable it, all other just disable */ int file_started; int file_finished; + int end_chunk; /* used for chunked transfer encoding. */ ! chunkqueue *write_queue; /* a large queue for HTTP response content [ file, mem ] */ ! chunkqueue *output_queue; /* a large queue for low-level write ( HTTP response ) [ file, mem ] */ chunkqueue *read_queue; /* a small queue for low-level read ( HTTP request ) [ mem ] */ chunkqueue *request_content_queue; /* takes request-content into tempfile if necessary [ tempfile, mem ]*/ int traffic_limit_reached; *************** *** 594,603 **** --- 597,607 ---- short int config_deprecated; short int config_unsupported; connections *conns; connections *joblist; + connections *joblist_prev; connections *fdwaitqueue; stat_cache *stat_cache; /** *** ../lighttpd-1.4.19/src/chunk.c 2006-10-04 21:26:23.000000000 +0800 --- src/chunk.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 14,24 **** --- 14,26 ---- #include #include #include + #include "server.h" #include "chunk.h" + #include "log.h" chunkqueue *chunkqueue_init(void) { chunkqueue *cq; cq = calloc(1, sizeof(*cq)); *************** *** 239,248 **** --- 241,260 ---- chunkqueue_append_chunk(cq, c); return 0; } + int chunkqueue_append_chunkqueue(chunkqueue *cq, chunkqueue *src) { + if(src == NULL) return 0; + chunkqueue_append_chunk(cq, src->first); + cq->last = src->last; + src->first = NULL; + src->last = NULL; + + return 0; + } + buffer * chunkqueue_get_prepend_buffer(chunkqueue *cq) { chunk *c; c = chunkqueue_get_unused_chunk(cq); *************** *** 398,402 **** --- 410,516 ---- } } return 0; } + + /** + * the HTTP chunk-API + * + * + */ + + static int chunk_encode_append_len(chunkqueue *cq, size_t len) { + size_t i, olen = len, j; + buffer *b; + + /*b = srv->tmp_chunk_len;*/ + /*b = buffer_init();*/ + b = chunkqueue_get_append_buffer(cq); + + if (len == 0) { + buffer_copy_string(b, "0"); + } else { + for (i = 0; i < 8 && len; i++) { + len >>= 4; + } + + /* i is the number of hex digits we have */ + buffer_prepare_copy(b, i + 1); + + for (j = i-1, len = olen; j+1 > 0; j--) { + b->ptr[j] = (len & 0xf) + (((len & 0xf) <= 9) ? '0' : 'a' - 10); + len >>= 4; + } + b->used = i; + b->ptr[b->used++] = '\0'; + } + + buffer_append_string(b, "\r\n"); + /* + chunkqueue_append_buffer(cq, b); + buffer_free(b); + */ + + return 0; + } + + + int chunk_encode_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) { + if (!cq) return -1; + if (len == 0) return 0; + + chunk_encode_append_len(cq, len); + + chunkqueue_append_file(cq, fn, offset, len); + + chunkqueue_append_mem(cq, "\r\n", 2 + 1); + + return 0; + } + + int chunk_encode_append_buffer(chunkqueue *cq, buffer *mem) { + if (!cq) return -1; + if (mem->used <= 1) return 0; + + chunk_encode_append_len(cq, mem->used - 1); + + chunkqueue_append_buffer(cq, mem); + + chunkqueue_append_mem(cq, "\r\n", 2 + 1); + + return 0; + } + + int chunk_encode_append_mem(chunkqueue *cq, const char * mem, size_t len) { + if (!cq) return -1; + if (len <= 1) return 0; + + chunk_encode_append_len(cq, len - 1); + + chunkqueue_append_mem(cq, mem, len); + + chunkqueue_append_mem(cq, "\r\n", 2 + 1); + + return 0; + } + + int chunk_encode_append_queue(chunkqueue *cq, chunkqueue *src) { + int len = chunkqueue_length(src); + if (!cq) return -1; + if (len == 0) return 0; + + chunk_encode_append_len(cq, len); + + chunkqueue_append_chunkqueue(cq, src); + + chunkqueue_append_mem(cq, "\r\n", 2 + 1); + + return 0; + } + + int chunk_encode_end(chunkqueue *cq) { + chunk_encode_append_len(cq, 0); + chunkqueue_append_mem(cq, "\r\n", 2 + 1); + return 0; + } + *** ../lighttpd-1.4.19/src/chunk.h 2006-10-04 21:26:23.000000000 +0800 --- src/chunk.h 2008-09-17 14:42:22.000000000 +0800 *************** *** 50,59 **** --- 50,60 ---- int chunkqueue_set_tempdirs(chunkqueue *c, array *tempdirs); int chunkqueue_append_file(chunkqueue *c, buffer *fn, off_t offset, off_t len); int chunkqueue_append_mem(chunkqueue *c, const char *mem, size_t len); int chunkqueue_append_buffer(chunkqueue *c, buffer *mem); int chunkqueue_append_buffer_weak(chunkqueue *c, buffer *mem); + int chunkqueue_append_chunkqueue(chunkqueue *cq, chunkqueue *src); int chunkqueue_prepend_buffer(chunkqueue *c, buffer *mem); buffer * chunkqueue_get_append_buffer(chunkqueue *c); buffer * chunkqueue_get_prepend_buffer(chunkqueue *c); chunk * chunkqueue_get_append_tempfile(chunkqueue *cq); *************** *** 65,70 **** --- 66,77 ---- void chunkqueue_free(chunkqueue *c); void chunkqueue_reset(chunkqueue *c); int chunkqueue_is_empty(chunkqueue *c); + int chunk_encode_append_mem(chunkqueue *cq, const char * mem, size_t len); + int chunk_encode_append_buffer(chunkqueue *cq, buffer *mem); + int chunk_encode_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len); + int chunk_encode_append_queue(chunkqueue *cq, chunkqueue *src); + int chunk_encode_end(chunkqueue *cq); + #endif *** ../lighttpd-1.4.19/src/connections.c 2008-09-17 14:34:08.000000000 +0800 --- src/connections.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 16,25 **** --- 16,26 ---- #include "request.h" #include "response.h" #include "network.h" #include "http_chunk.h" + #include "chunk.h" #include "stat_cache.h" #include "joblist.h" #include "plugin.h" *************** *** 144,153 **** --- 145,160 ---- connection_set_state(srv, con, CON_STATE_CONNECT); return 0; } + int connection_queue_is_empty(connection *con) { + if(!chunkqueue_is_empty(con->write_queue)) return 0; + if(!chunkqueue_is_empty(con->output_queue)) return 0; + return 1; + } + #if 0 static void dump_packet(const unsigned char *data, size_t len) { size_t i, j; if (len == 0) return; *************** *** 404,413 **** --- 411,421 ---- con->http_status = 200; con->file_finished = 1; chunkqueue_reset(con->write_queue); + chunkqueue_reset(con->output_queue); } break; default: switch(con->http_status) { case 400: /* bad request */ *************** *** 517,531 **** --- 525,554 ---- default: /* disable chunked encoding again as we have no body */ con->response.transfer_encoding &= ~HTTP_TRANSFER_ENCODING_CHUNKED; con->parsed_response &= ~HTTP_CONTENT_LENGTH; chunkqueue_reset(con->write_queue); + chunkqueue_reset(con->output_queue); con->file_finished = 1; break; } + /* Allow filter plugins to change response headers before they are written. */ + switch(plugins_call_handle_response_start(srv, con)) { + case HANDLER_GO_ON: + case HANDLER_FINISHED: + /* response start is finished */ + break; + default: + /* something strange happend */ + log_error_write(srv, __FILE__, __LINE__, "s", "Filter plugin failed."); + connection_set_state(srv, con, CON_STATE_ERROR); + joblist_append(srv, con); + break; + } + if (con->file_finished) { /* we have all the content and chunked encoding is not used, set a content-length */ if ((!(con->parsed_response & HTTP_CONTENT_LENGTH)) && (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) == 0) { *************** *** 593,616 **** * without the content */ con->file_finished = 1; chunkqueue_reset(con->write_queue); con->response.transfer_encoding &= ~HTTP_TRANSFER_ENCODING_CHUNKED; } http_response_write_header(srv, con); return 0; } static int connection_handle_write(server *srv, connection *con) { ! switch(network_write_chunkqueue(srv, con, con->write_queue)) { case 0: ! if (con->file_finished) { connection_set_state(srv, con, CON_STATE_RESPONSE_END); joblist_append(srv, con); } break; case -1: /* error on our side */ log_error_write(srv, __FILE__, __LINE__, "sd", "connection closed: write failed on fd", con->fd); --- 616,686 ---- * without the content */ con->file_finished = 1; chunkqueue_reset(con->write_queue); + chunkqueue_reset(con->output_queue); con->response.transfer_encoding &= ~HTTP_TRANSFER_ENCODING_CHUNKED; } http_response_write_header(srv, con); return 0; } static int connection_handle_write(server *srv, connection *con) { ! int finished = 0; ! int len; ! ! /* Allow filter plugins to modify response conent */ ! switch(plugins_call_handle_response_filter(srv, con)) { ! case HANDLER_GO_ON: ! finished = con->file_finished; ! /* response content not changed */ ! break; ! case HANDLER_COMEBACK: ! /* response filter has more work */ ! finished = 0; ! break; ! case HANDLER_FINISHED: ! /* response filter is finished */ ! finished = 1; ! break; ! default: ! /* something strange happend */ ! log_error_write(srv, __FILE__, __LINE__, "s", "Filter plugin failed."); ! connection_set_state(srv, con, CON_STATE_ERROR); ! joblist_append(srv, con); ! finished = 1; ! break; ! } ! ! /* move chunks from write_queue to output_queue. */ ! if (con->request.http_method == HTTP_METHOD_HEAD) { ! chunkqueue_reset(con->write_queue); ! } else { ! len = chunkqueue_length(con->write_queue); ! if(con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) { ! chunk_encode_append_queue(con->output_queue, con->write_queue); ! if(finished && !con->end_chunk) { ! con->end_chunk = 1; ! chunk_encode_end(con->output_queue); ! } ! } else { ! chunkqueue_append_chunkqueue(con->output_queue, con->write_queue); ! } ! con->write_queue->bytes_out += len; ! } ! /* write chunks from output_queue to network */ ! switch(network_write_chunkqueue(srv, con, con->output_queue)) { case 0: ! if (finished) { connection_set_state(srv, con, CON_STATE_RESPONSE_END); joblist_append(srv, con); + } else { + /* not finished yet -> WRITE */ + con->is_writable = 1; } break; case -1: /* error on our side */ log_error_write(srv, __FILE__, __LINE__, "sd", "connection closed: write failed on fd", con->fd); *************** *** 676,685 **** --- 746,756 ---- CLEAN(error_handler); CLEAN(dst_addr_buf); #undef CLEAN con->write_queue = chunkqueue_init(); + con->output_queue = chunkqueue_init(); con->read_queue = chunkqueue_init(); con->request_content_queue = chunkqueue_init(); chunkqueue_set_tempdirs(con->request_content_queue, srv->srvconf.upload_tempdirs); con->request.headers = array_init(); *************** *** 704,713 **** --- 775,785 ---- connection *con = conns->ptr[i]; connection_reset(srv, con); chunkqueue_free(con->write_queue); + chunkqueue_free(con->output_queue); chunkqueue_free(con->read_queue); chunkqueue_free(con->request_content_queue); array_free(con->request.headers); array_free(con->response.headers); array_free(con->environment); *************** *** 758,768 **** --- 830,843 ---- con->is_readable = 1; con->is_writable = 1; con->http_status = 0; con->file_finished = 0; con->file_started = 0; + con->end_chunk = 0; con->got_response = 0; + // con->use_cache_file = 0; + // con->write_cache_file = 0; con->parsed_response = 0; con->bytes_written = 0; con->bytes_written_cur_second = 0; *************** *** 828,837 **** --- 903,913 ---- array_reset(con->request.headers); array_reset(con->response.headers); array_reset(con->environment); chunkqueue_reset(con->write_queue); + chunkqueue_reset(con->output_queue); chunkqueue_reset(con->request_content_queue); /* the plugins should cleanup themself */ for (i = 0; i < srv->plugins.used; i++) { plugin *p = ((plugin **)(srv->plugins.ptr))[i]; *************** *** 1222,1232 **** con->state == CON_STATE_READ_POST) { connection_handle_read_state(srv, con); } if (con->state == CON_STATE_WRITE && - !chunkqueue_is_empty(con->write_queue) && con->is_writable) { if (-1 == connection_handle_write(srv, con)) { connection_set_state(srv, con, CON_STATE_ERROR); --- 1298,1307 ---- *************** *** 1639,1657 **** log_error_write(srv, __FILE__, __LINE__, "sds", "state for fd", con->fd, connection_get_state(con->state)); } /* only try to write if we have something in the queue */ - if (!chunkqueue_is_empty(con->write_queue)) { #if 0 log_error_write(srv, __FILE__, __LINE__, "dsd", con->fd, "packets to write:", ! con->write_queue->used); ! #endif } ! if (!chunkqueue_is_empty(con->write_queue) && con->is_writable) { if (-1 == connection_handle_write(srv, con)) { log_error_write(srv, __FILE__, __LINE__, "ds", con->fd, "handle write failed."); connection_set_state(srv, con, CON_STATE_ERROR); --- 1714,1732 ---- log_error_write(srv, __FILE__, __LINE__, "sds", "state for fd", con->fd, connection_get_state(con->state)); } /* only try to write if we have something in the queue */ #if 0 + if (!connection_queue_is_empty(con)) { log_error_write(srv, __FILE__, __LINE__, "dsd", con->fd, "packets to write:", ! con->output_queue->used); } ! #endif ! if (con->is_writable) { if (-1 == connection_handle_write(srv, con)) { log_error_write(srv, __FILE__, __LINE__, "ds", con->fd, "handle write failed."); connection_set_state(srv, con, CON_STATE_ERROR); *************** *** 1787,1799 **** case CON_STATE_WRITE: /* request write-fdevent only if we really need it * - if we have data to write * - if the socket is not writable yet */ ! if (!chunkqueue_is_empty(con->write_queue) && ! (con->is_writable == 0) && ! (con->traffic_limit_reached == 0)) { fdevent_event_add(srv->ev, &(con->fde_ndx), con->fd, FDEVENT_OUT); } else { fdevent_event_del(srv->ev, &(con->fde_ndx), con->fd); } break; --- 1862,1874 ---- case CON_STATE_WRITE: /* request write-fdevent only if we really need it * - if we have data to write * - if the socket is not writable yet */ ! if ((con->is_writable == 0) && ! (con->traffic_limit_reached == 0) && ! !connection_queue_is_empty(con)) { fdevent_event_add(srv->ev, &(con->fde_ndx), con->fd, FDEVENT_OUT); } else { fdevent_event_del(srv->ev, &(con->fde_ndx), con->fd); } break; *** ../lighttpd-1.4.19/src/http_chunk.c 2006-10-04 21:26:23.000000000 +0800 --- src/http_chunk.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 56,95 **** if (!con) return -1; cq = con->write_queue; - if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) { - http_chunk_append_len(srv, con, len); - } chunkqueue_append_file(cq, fn, offset, len); - if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED && len > 0) { - chunkqueue_append_mem(cq, "\r\n", 2 + 1); - } - return 0; } int http_chunk_append_buffer(server *srv, connection *con, buffer *mem) { chunkqueue *cq; if (!con) return -1; cq = con->write_queue; - if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) { - http_chunk_append_len(srv, con, mem->used - 1); - } chunkqueue_append_buffer(cq, mem); - if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED && mem->used > 0) { - chunkqueue_append_mem(cq, "\r\n", 2 + 1); - } - return 0; } int http_chunk_append_mem(server *srv, connection *con, const char * mem, size_t len) { chunkqueue *cq; --- 56,81 ---- *************** *** 97,124 **** if (!con) return -1; cq = con->write_queue; if (len == 0) { - if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) { - chunkqueue_append_mem(cq, "0\r\n\r\n", 5 + 1); - } else { - chunkqueue_append_mem(cq, "", 1); - } return 0; } - if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) { - http_chunk_append_len(srv, con, len - 1); - } - chunkqueue_append_mem(cq, mem, len); - if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) { - chunkqueue_append_mem(cq, "\r\n", 2 + 1); - } - return 0; } off_t http_chunkqueue_length(server *srv, connection *con) { --- 83,97 ---- *** ../lighttpd-1.4.19/src/joblist.c 2006-10-04 21:26:23.000000000 +0800 --- src/joblist.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 5,14 **** --- 5,15 ---- #include "joblist.h" #include "log.h" int joblist_append(server *srv, connection *con) { if (con->in_joblist) return 0; + con->in_joblist = 1; if (srv->joblist->size == 0) { srv->joblist->size = 16; srv->joblist->ptr = malloc(sizeof(*srv->joblist->ptr) * srv->joblist->size); } else if (srv->joblist->used == srv->joblist->size) { *** ../lighttpd-1.4.19/src/Makefile.am 2008-02-27 00:21:09.000000000 +0800 --- src/Makefile.am 2008-09-17 14:42:22.000000000 +0800 *************** *** 239,248 **** --- 239,253 ---- lib_LTLIBRARIES += mod_accesslog.la mod_accesslog_la_SOURCES = mod_accesslog.c mod_accesslog_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined mod_accesslog_la_LIBADD = $(common_libadd) + lib_LTLIBRARIES += mod_deflate.la + mod_deflate_la_SOURCES = mod_deflate.c + mod_deflate_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined + mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(common_libadd) + hdr = server.h buffer.h network.h log.h keyvalue.h \ response.h request.h fastcgi.h chunk.h \ settings.h http_chunk.h http_auth_digest.h \ md5.h http_auth.h stream.h \ *** ../lighttpd-1.4.19/src/Makefile.in 2008-03-11 05:46:10.000000000 +0800 --- src/Makefile.in 2008-09-17 14:42:22.000000000 +0800 *************** *** 158,167 **** --- 158,174 ---- am_mod_compress_la_OBJECTS = mod_compress.lo mod_compress_la_OBJECTS = $(am_mod_compress_la_OBJECTS) mod_compress_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ $(mod_compress_la_LDFLAGS) $(LDFLAGS) -o $@ + mod_deflate_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) + am_mod_deflate_la_OBJECTS = mod_deflate.lo + mod_deflate_la_OBJECTS = $(am_mod_deflate_la_OBJECTS) + mod_deflate_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(mod_deflate_la_LDFLAGS) $(LDFLAGS) -o $@ mod_dirlisting_la_DEPENDENCIES = $(am__DEPENDENCIES_2) \ $(am__DEPENDENCIES_1) am_mod_dirlisting_la_OBJECTS = mod_dirlisting.lo mod_dirlisting_la_OBJECTS = $(am_mod_dirlisting_la_OBJECTS) mod_dirlisting_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \ *************** *** 397,406 **** --- 404,414 ---- $(LDFLAGS) -o $@ SOURCES = $(liblightcomp_la_SOURCES) $(mod_access_la_SOURCES) \ $(mod_accesslog_la_SOURCES) $(mod_alias_la_SOURCES) \ $(mod_auth_la_SOURCES) $(mod_cgi_la_SOURCES) \ $(mod_cml_la_SOURCES) $(mod_compress_la_SOURCES) \ + $(mod_deflate_la_SOURCES) \ $(mod_dirlisting_la_SOURCES) $(mod_evasive_la_SOURCES) \ $(mod_evhost_la_SOURCES) $(mod_expire_la_SOURCES) \ $(mod_extforward_la_SOURCES) $(mod_fastcgi_la_SOURCES) \ $(mod_flv_streaming_la_SOURCES) $(mod_indexfile_la_SOURCES) \ $(mod_magnet_la_SOURCES) $(mod_mysql_vhost_la_SOURCES) \ *************** *** 612,622 **** mod_dirlisting.la mod_indexfile.la mod_setenv.la mod_alias.la \ mod_userdir.la mod_rrdtool.la mod_usertrack.la mod_proxy.la \ mod_ssi.la mod_secdownload.la mod_expire.la mod_evhost.la \ mod_simple_vhost.la mod_fastcgi.la mod_extforward.la \ mod_access.la mod_compress.la mod_auth.la mod_rewrite.la \ ! mod_redirect.la mod_status.la mod_accesslog.la @NO_RDYNAMIC_TRUE@liblightcomp_la_SOURCES = $(common_src) @NO_RDYNAMIC_TRUE@liblightcomp_la_CFLAGS = $(AM_CFLAGS) @NO_RDYNAMIC_TRUE@liblightcomp_la_LDFLAGS = -avoid-version -no-undefined @NO_RDYNAMIC_TRUE@liblightcomp_la_LIBADD = $(PCRE_LIB) $(SSL_LIB) $(FAM_LIBS) @NO_RDYNAMIC_FALSE@common_libadd = --- 620,630 ---- mod_dirlisting.la mod_indexfile.la mod_setenv.la mod_alias.la \ mod_userdir.la mod_rrdtool.la mod_usertrack.la mod_proxy.la \ mod_ssi.la mod_secdownload.la mod_expire.la mod_evhost.la \ mod_simple_vhost.la mod_fastcgi.la mod_extforward.la \ mod_access.la mod_compress.la mod_auth.la mod_rewrite.la \ ! mod_redirect.la mod_status.la mod_accesslog.la mod_deflate.la @NO_RDYNAMIC_TRUE@liblightcomp_la_SOURCES = $(common_src) @NO_RDYNAMIC_TRUE@liblightcomp_la_CFLAGS = $(AM_CFLAGS) @NO_RDYNAMIC_TRUE@liblightcomp_la_LDFLAGS = -avoid-version -no-undefined @NO_RDYNAMIC_TRUE@liblightcomp_la_LIBADD = $(PCRE_LIB) $(SSL_LIB) $(FAM_LIBS) @NO_RDYNAMIC_FALSE@common_libadd = *************** *** 719,728 **** --- 727,739 ---- mod_status_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined mod_status_la_LIBADD = $(common_libadd) mod_accesslog_la_SOURCES = mod_accesslog.c mod_accesslog_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined mod_accesslog_la_LIBADD = $(common_libadd) + mod_deflate_la_SOURCES = mod_deflate.c + mod_deflate_la_LDFLAGS = -module -export-dynamic -no-undefined + mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(common_libadd) hdr = server.h buffer.h network.h log.h keyvalue.h \ response.h request.h fastcgi.h chunk.h \ settings.h http_chunk.h http_auth_digest.h \ md5.h http_auth.h stream.h \ fdevent.h connections.h base.h stat_cache.h \ *************** *** 830,839 **** --- 841,852 ---- $(mod_cgi_la_LINK) -rpath $(libdir) $(mod_cgi_la_OBJECTS) $(mod_cgi_la_LIBADD) $(LIBS) mod_cml.la: $(mod_cml_la_OBJECTS) $(mod_cml_la_DEPENDENCIES) $(mod_cml_la_LINK) -rpath $(libdir) $(mod_cml_la_OBJECTS) $(mod_cml_la_LIBADD) $(LIBS) mod_compress.la: $(mod_compress_la_OBJECTS) $(mod_compress_la_DEPENDENCIES) $(mod_compress_la_LINK) -rpath $(libdir) $(mod_compress_la_OBJECTS) $(mod_compress_la_LIBADD) $(LIBS) + mod_deflate.la: $(mod_deflate_la_OBJECTS) $(mod_deflate_la_DEPENDENCIES) + $(mod_deflate_la_LINK) -rpath $(libdir) $(mod_deflate_la_LDFLAGS) $(mod_deflate_la_OBJECTS) $(mod_deflate_la_LIBADD) $(LIBS) mod_dirlisting.la: $(mod_dirlisting_la_OBJECTS) $(mod_dirlisting_la_DEPENDENCIES) $(mod_dirlisting_la_LINK) -rpath $(libdir) $(mod_dirlisting_la_OBJECTS) $(mod_dirlisting_la_LIBADD) $(LIBS) mod_evasive.la: $(mod_evasive_la_OBJECTS) $(mod_evasive_la_DEPENDENCIES) $(mod_evasive_la_LINK) -rpath $(libdir) $(mod_evasive_la_OBJECTS) $(mod_evasive_la_LIBADD) $(LIBS) mod_evhost.la: $(mod_evhost_la_OBJECTS) $(mod_evhost_la_DEPENDENCIES) *************** *** 1048,1057 **** --- 1061,1071 ---- @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_cgi.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_cml_la-mod_cml.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_cml_la-mod_cml_funcs.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_cml_la-mod_cml_lua.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_compress.Plo@am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_deflate.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_dirlisting.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_evasive.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_evhost.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_expire.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mod_extforward.Plo@am__quote@ *** ../lighttpd-1.4.19/src/mod_deflate.c 1970-01-01 08:00:00.000000000 +0800 --- src/mod_deflate.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 0 **** --- 1,1420 ---- + /* bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate path + * + * new module option: + * deflate.nocompress-url = "^/nocompressurl/" # pcre regex which don't compress + * + * Bug fix and new features: + * 1) fix loop bug when content-length is bigger than work-block-size*k + * 2) prevent compress on buggy http 1.0 client with Accept Encoding: gzip, deflate + * 3) fix bug with chunk transfer encoding (under mod_fastcgi+php environment) + * + * deflate.sync-flush = "enable" is buggy on chunk encoding transfer. Use it carefully, + */ + #include + #include + + #include + #include + #include + #include + #include + #include + #include + #include + + #if defined(HAVE_PCRE_H) + #include + #endif + + #include "base.h" + #include "log.h" + #include "buffer.h" + #include "response.h" + #include "joblist.h" + #include "stat_cache.h" + + #include "plugin.h" + + #include "crc32.h" + #include "etag.h" + #include "inet_ntop_cache.h" + + #if defined HAVE_ZLIB_H && defined HAVE_LIBZ + # define USE_ZLIB + # include + #else + # define Z_DEFAULT_COMPRESSION 1 + #endif + + #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 + # define USE_BZ2LIB + /* we don't need stdio interface */ + # define BZ_NO_STDIO + # include + #endif + + #include "sys-mmap.h" + + /* request: accept-encoding */ + #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) + #define HTTP_ACCEPT_ENCODING_GZIP BV(1) + #define HTTP_ACCEPT_ENCODING_DEFLATE BV(2) + #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3) + #define HTTP_ACCEPT_ENCODING_BZIP2 BV(4) + + #define KByte * 1024 + #define MByte * 1024 KByte + #define GByte * 1024 MByte + + typedef struct { + unsigned short debug; + unsigned short enabled; + unsigned short bzip2; + unsigned short sync_flush; + unsigned short output_buffer_size; + unsigned short min_compress_size; + unsigned short work_block_size; + short mem_level; + short compression_level; + short window_size; + array *mimetypes; + buffer *nocompress_url; + #if defined(HAVE_PCRE_H) + pcre *nocompress_regex; + #endif + } plugin_config; + + typedef struct { + PLUGIN_DATA; + buffer *tmp_buf; + + plugin_config **config_storage; + plugin_config conf; + } plugin_data; + + typedef struct { + int bytes_in; + int bytes_out; + chunkqueue *in_queue; + buffer *output; + /* compression type & state */ + int compression_type; + int stream_open; + #ifdef USE_ZLIB + unsigned long crc; + z_stream z; + unsigned short gzip_header; + #endif + #ifdef USE_BZ2LIB + bz_stream bz; + #endif + plugin_data *plugin_data; + } handler_ctx; + + static handler_ctx *handler_ctx_init() { + handler_ctx *hctx; + + hctx = calloc(1, sizeof(*hctx)); + hctx->in_queue = chunkqueue_init(); + + return hctx; + } + + static void handler_ctx_free(handler_ctx *hctx) { + chunkqueue_free(hctx->in_queue); + free(hctx); + } + + INIT_FUNC(mod_deflate_init) { + plugin_data *p; + + p = calloc(1, sizeof(*p)); + + p->tmp_buf = buffer_init(); + + return p; + } + + FREE_FUNC(mod_deflate_free) { + plugin_data *p = p_d; + + UNUSED(srv); + + if (!p) return HANDLER_GO_ON; + + if (p->config_storage) { + size_t i; + for (i = 0; i < srv->config_context->used; i++) { + plugin_config *s = p->config_storage[i]; + + if (!s) continue; + + array_free(s->mimetypes); + buffer_free(s->nocompress_url); + #if defined(HAVE_PCRE_H) + if (s->nocompress_regex) pcre_free(s->nocompress_regex); + #endif + free(s); + } + free(p->config_storage); + } + + buffer_free(p->tmp_buf); + + free(p); + + return HANDLER_GO_ON; + } + + SETDEFAULTS_FUNC(mod_deflate_setdefaults) { + plugin_data *p = p_d; + size_t i = 0; + + config_values_t cv[] = { + { "deflate.output-buffer-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.mimetypes", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.compression-level", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.mem-level", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.window-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.min-compress-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.work-block-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.enabled", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.debug", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.bzip2", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.sync-flush", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, + { "deflate.nocompress-url", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION }, + { NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET } + }; + + p->config_storage = calloc(1, srv->config_context->used * sizeof(specific_config *)); + + for (i = 0; i < srv->config_context->used; i++) { + plugin_config *s; + #if defined(HAVE_PCRE_H) + const char *errptr; + int erroff; + #endif + + s = calloc(1, sizeof(plugin_config)); + s->enabled = 1; + s->bzip2 = 1; + s->sync_flush = 0; + s->debug = 0; + s->output_buffer_size = 0; + s->mem_level = 9; + s->window_size = 15; + s->min_compress_size = 0; + s->work_block_size = 2048; + s->compression_level = Z_DEFAULT_COMPRESSION; + s->mimetypes = array_init(); + s->nocompress_url = buffer_init(); + #if defined(HAVE_PCRE_H) + s->nocompress_regex = NULL; + #endif + + cv[0].destination = &(s->output_buffer_size); + cv[1].destination = s->mimetypes; + cv[2].destination = &(s->compression_level); + cv[3].destination = &(s->mem_level); + cv[4].destination = &(s->window_size); + cv[5].destination = &(s->min_compress_size); + cv[6].destination = &(s->work_block_size); + cv[7].destination = &(s->enabled); + cv[8].destination = &(s->debug); + cv[9].destination = &(s->bzip2); + cv[10].destination = &(s->sync_flush); + cv[11].destination = s->nocompress_url; + + p->config_storage[i] = s; + + if (0 != config_insert_values_global(srv, ((data_config *)srv->config_context->data[i])->value, cv)) { + return HANDLER_ERROR; + } + + #if defined(HAVE_PCRE_H) + if (!buffer_is_empty(s->nocompress_url)) { + if (NULL == (s->nocompress_regex = pcre_compile(s->nocompress_url->ptr, + 0, &errptr, &erroff, NULL))) { + + log_error_write(srv, __FILE__, __LINE__, "sbss", + "compiling regex for nocompress-url failed:", + s->nocompress_url, "pos:", erroff); + return HANDLER_ERROR; + } + } + #endif + if((s->compression_level < 1 || s->compression_level > 9) && + s->compression_level != Z_DEFAULT_COMPRESSION) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "compression-level must be between 1 and 9:", s->compression_level); + return HANDLER_ERROR; + } + + if(s->mem_level < 1 || s->mem_level > 9) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "mem-level must be between 1 and 9:", s->mem_level); + return HANDLER_ERROR; + } + + if(s->window_size < 1 || s->window_size > 15) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "window-size must be between 1 and 15:", s->window_size); + return HANDLER_ERROR; + } + s->window_size = 0 - s->window_size; + + if(s->sync_flush) { + s->output_buffer_size = 0; + } + } + + return HANDLER_GO_ON; + + } + + #ifdef USE_ZLIB + /* Copied gzip_header from apache 2.2's mod_deflate.c */ + /* RFC 1952 Section 2.3 defines the gzip header: + * + * +---+---+---+---+---+---+---+---+---+---+ + * |ID1|ID2|CM |FLG| MTIME |XFL|OS | + * +---+---+---+---+---+---+---+---+---+---+ + */ + static const char gzip_header[10] = + { '\037', '\213', Z_DEFLATED, 0, + 0, 0, 0, 0, /* mtime */ + 0, 0x03 /* Unix OS_CODE */ + }; + static int stream_deflate_init(server *srv, connection *con, handler_ctx *hctx) { + plugin_data *p = hctx->plugin_data; + z_stream *z; + + UNUSED(srv); + UNUSED(con); + + z = &(hctx->z); + z->zalloc = Z_NULL; + z->zfree = Z_NULL; + z->opaque = Z_NULL; + z->total_in = 0; + z->total_out = 0; + z->next_out = NULL; + z->avail_out = 0; + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "output-buffer-size:", p->conf.output_buffer_size); + log_error_write(srv, __FILE__, __LINE__, "sd", + "compression-level:", p->conf.compression_level); + log_error_write(srv, __FILE__, __LINE__, "sd", + "mem-level:", p->conf.mem_level); + log_error_write(srv, __FILE__, __LINE__, "sd", + "window-size:", p->conf.window_size); + log_error_write(srv, __FILE__, __LINE__, "sd", + "min-compress-size:", p->conf.min_compress_size); + log_error_write(srv, __FILE__, __LINE__, "sd", + "work-block-size:", p->conf.work_block_size); + } + if (Z_OK != deflateInit2(z, + p->conf.compression_level, + Z_DEFLATED, + p->conf.window_size, /* supress zlib-header */ + p->conf.mem_level, + Z_DEFAULT_STRATEGY)) { + return -1; + } + hctx->stream_open = 1; + + return 0; + } + + static int stream_deflate_compress(server *srv, connection *con, handler_ctx *hctx, unsigned char *start, off_t st_size) { + plugin_data *p = hctx->plugin_data; + z_stream *z; + int len; + int in = 0, out = 0; + + UNUSED(srv); + z = &(hctx->z); + + if(z->next_out == NULL) { + z->next_out = (unsigned char *)hctx->output->ptr; + z->avail_out = hctx->output->size; + } + + if(hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) { + if(hctx->gzip_header == 0) { + hctx->gzip_header = 1; + /* copy gzip header into output buffer */ + buffer_copy_memory(hctx->output, gzip_header, sizeof(gzip_header)); + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "gzip_header len=", sizeof(gzip_header)); + } + /* initialize crc32 */ + hctx->crc = crc32(0L, Z_NULL, 0); + z->next_out = (unsigned char *)(hctx->output->ptr + sizeof(gzip_header)); + z->avail_out = hctx->output->size - sizeof(gzip_header); + } + hctx->crc = crc32(hctx->crc, start, st_size); + } + + z->next_in = start; + z->avail_in = st_size; + hctx->bytes_in += st_size; + + /* compress data */ + in = z->avail_in; + do { + if (Z_OK != deflate(z, Z_NO_FLUSH)) { + deflateEnd(z); + hctx->stream_open = 0; + return -1; + } + + if(z->avail_out == 0 || z->avail_in > 0) { + len = hctx->output->size - z->avail_out; + hctx->bytes_out += len; + out += len; + chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); + z->next_out = (unsigned char *)hctx->output->ptr; + z->avail_out = hctx->output->size; + } + } while (z->avail_in > 0); + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sdsd", + "compress: in=", in, ", out=", out); + } + return 0; + } + + static int stream_deflate_flush(server *srv, connection *con, handler_ctx *hctx, int end) { + plugin_data *p = hctx->plugin_data; + z_stream *z; + int len; + int rc = 0; + int done; + int flush = 1; + int in = 0, out = 0; + + UNUSED(srv); + + z = &(hctx->z); + + if(z->next_out == NULL) { + z->next_out = (unsigned char *)hctx->output->ptr; + z->avail_out = hctx->output->size; + } + /* compress data */ + in = z->avail_in; + do { + done = 1; + if(end) { + rc = deflate(z, Z_FINISH); + if (rc == Z_OK) { + done = 0; + } else if (rc != Z_STREAM_END) { + deflateEnd(z); + hctx->stream_open = 0; + return -1; + } + } else { + if(p->conf.sync_flush) { + rc = deflate(z, Z_SYNC_FLUSH); + } else if(z->avail_in > 0) { + if(p->conf.output_buffer_size > 0) flush = 0; + rc = deflate(z, Z_NO_FLUSH); + } else { + if(p->conf.output_buffer_size > 0) flush = 0; + rc = Z_OK; + } + if (rc != Z_OK) { + deflateEnd(z); + hctx->stream_open = 0; + return -1; + } + } + + len = hctx->output->size - z->avail_out; + if(z->avail_out == 0 || (flush && len > 0)) { + hctx->bytes_out += len; + out += len; + chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); + z->next_out = (unsigned char *)hctx->output->ptr; + z->avail_out = hctx->output->size; + } + } while (z->avail_in != 0 || !done); + + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sdsd", + "flush: in=", in, ", out=", out); + } + if(p->conf.sync_flush) { + z->next_out = NULL; + z->avail_out = 0; + } + return 0; + } + + static int stream_deflate_end(server *srv, connection *con, handler_ctx *hctx) { + plugin_data *p = hctx->plugin_data; + z_stream *z; + int rc; + + UNUSED(srv); + + z = &(hctx->z); + if(!hctx->stream_open) return 0; + hctx->stream_open = 0; + + if(hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP && hctx->bytes_out > 0 && + (unsigned int )hctx->bytes_out >= sizeof(gzip_header)) { + /* write gzip footer */ + unsigned char c[8]; + + c[0] = (hctx->crc >> 0) & 0xff; + c[1] = (hctx->crc >> 8) & 0xff; + c[2] = (hctx->crc >> 16) & 0xff; + c[3] = (hctx->crc >> 24) & 0xff; + c[4] = (z->total_in >> 0) & 0xff; + c[5] = (z->total_in >> 8) & 0xff; + c[6] = (z->total_in >> 16) & 0xff; + c[7] = (z->total_in >> 24) & 0xff; + /* append footer to write_queue */ + chunkqueue_append_mem(con->write_queue, (char *)c, 9); + hctx->bytes_out += 8; + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "gzip_footer len=", 8); + } + } + + if ((rc = deflateEnd(z)) != Z_OK) { + if(rc == Z_DATA_ERROR) return 0; + if(z->msg != NULL) { + log_error_write(srv, __FILE__, __LINE__, "sdss", + "deflateEnd error ret=", rc, ", msg=", z->msg); + } else { + log_error_write(srv, __FILE__, __LINE__, "sd", + "deflateEnd error ret=", rc); + } + return -1; + } + return 0; + } + + #endif + + #ifdef USE_BZ2LIB + static int stream_bzip2_init(server *srv, connection *con, handler_ctx *hctx) { + plugin_data *p = hctx->plugin_data; + bz_stream *bz; + + UNUSED(srv); + UNUSED(con); + + bz = &(hctx->bz); + bz->bzalloc = NULL; + bz->bzfree = NULL; + bz->opaque = NULL; + bz->total_in_lo32 = 0; + bz->total_in_hi32 = 0; + bz->total_out_lo32 = 0; + bz->total_out_hi32 = 0; + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "output-buffer-size:", p->conf.output_buffer_size); + log_error_write(srv, __FILE__, __LINE__, "sd", + "compression-level:", p->conf.compression_level); + log_error_write(srv, __FILE__, __LINE__, "sd", + "mem-level:", p->conf.mem_level); + log_error_write(srv, __FILE__, __LINE__, "sd", + "window-size:", p->conf.window_size); + log_error_write(srv, __FILE__, __LINE__, "sd", + "min-compress-size:", p->conf.min_compress_size); + log_error_write(srv, __FILE__, __LINE__, "sd", + "work-block-size:", p->conf.work_block_size); + } + if (BZ_OK != BZ2_bzCompressInit(bz, + p->conf.compression_level, /* blocksize = 900k */ + 0, /* no output */ + 30)) { /* workFactor: default */ + return -1; + } + hctx->stream_open = 1; + + return 0; + } + + static int stream_bzip2_compress(server *srv, connection *con, handler_ctx *hctx, unsigned char *start, off_t st_size) { + plugin_data *p = hctx->plugin_data; + bz_stream *bz; + int len; + int rc; + int in = 0, out = 0; + + UNUSED(srv); + + bz = &(hctx->bz); + + if(bz->next_out == NULL) { + bz->next_out = hctx->output->ptr; + bz->avail_out = hctx->output->size; + } + + bz->next_in = (char *)start; + bz->avail_in = st_size; + hctx->bytes_in += st_size; + + /* compress data */ + in = bz->avail_in; + do { + rc = BZ2_bzCompress(bz, BZ_RUN); + if (rc != BZ_RUN_OK) { + BZ2_bzCompressEnd(bz); + hctx->stream_open = 0; + return -1; + } + + if(bz->avail_out == 0 || bz->avail_in > 0) { + len = hctx->output->size - bz->avail_out; + hctx->bytes_out += len; + out += len; + chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); + bz->next_out = hctx->output->ptr; + bz->avail_out = hctx->output->size; + } + } while (bz->avail_in > 0); + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sdsd", + "compress: in=", in, ", out=", out); + } + return 0; + } + + static int stream_bzip2_flush(server *srv, connection *con, handler_ctx *hctx, int end) { + plugin_data *p = hctx->plugin_data; + bz_stream *bz; + int len; + int rc; + int done; + int flush = 1; + int in = 0, out = 0; + + UNUSED(srv); + + bz = &(hctx->bz); + + if(bz->next_out == NULL) { + bz->next_out = hctx->output->ptr; + bz->avail_out = hctx->output->size; + } + /* compress data */ + in = bz->avail_in; + do { + done = 1; + if(end) { + rc = BZ2_bzCompress(bz, BZ_FINISH); + if (rc == BZ_FINISH_OK) { + done = 0; + } else if (rc != BZ_STREAM_END) { + BZ2_bzCompressEnd(bz); + hctx->stream_open = 0; + return -1; + } + } else if(bz->avail_in > 0) { + rc = BZ2_bzCompress(bz, BZ_RUN); + if (rc != BZ_RUN_OK) { + BZ2_bzCompressEnd(bz); + hctx->stream_open = 0; + return -1; + } + if(p->conf.output_buffer_size > 0) flush = 0; + } + + len = hctx->output->size - bz->avail_out; + if(bz->avail_out == 0 || (flush && len > 0)) { + hctx->bytes_out += len; + out += len; + chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); + bz->next_out = hctx->output->ptr; + bz->avail_out = hctx->output->size; + } + } while (bz->avail_in != 0 || !done); + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sdsd", + "flush: in=", in, ", out=", out); + } + if(p->conf.sync_flush) { + bz->next_out = NULL; + bz->avail_out = 0; + } + return 0; + } + + static int stream_bzip2_end(server *srv, connection *con, handler_ctx *hctx) { + plugin_data *p = hctx->plugin_data; + bz_stream *bz; + int rc; + + UNUSED(p); + UNUSED(con); + + bz = &(hctx->bz); + if(!hctx->stream_open) return 0; + hctx->stream_open = 0; + + if ((rc = BZ2_bzCompressEnd(bz)) != BZ_OK) { + if(rc == BZ_DATA_ERROR) return 0; + log_error_write(srv, __FILE__, __LINE__, "sd", + "BZ2_bzCompressEnd error ret=", rc); + return -1; + } + return 0; + } + + #endif + + static int mod_deflate_compress(server *srv, connection *con, handler_ctx *hctx, unsigned char *start, off_t st_size) { + int ret = -1; + if(st_size == 0) return 0; + switch(hctx->compression_type) { + #ifdef USE_ZLIB + case HTTP_ACCEPT_ENCODING_GZIP: + case HTTP_ACCEPT_ENCODING_DEFLATE: + ret = stream_deflate_compress(srv, con, hctx, start, st_size); + break; + #endif + #ifdef USE_BZ2LIB + case HTTP_ACCEPT_ENCODING_BZIP2: + ret = stream_bzip2_compress(srv, con, hctx, start, st_size); + break; + #endif + default: + ret = -1; + break; + } + + return ret; + } + + static int mod_deflate_stream_flush(server *srv, connection *con, handler_ctx *hctx, int end) { + int ret = -1; + if(hctx->bytes_in == 0) return 0; + switch(hctx->compression_type) { + #ifdef USE_ZLIB + case HTTP_ACCEPT_ENCODING_GZIP: + case HTTP_ACCEPT_ENCODING_DEFLATE: + ret = stream_deflate_flush(srv, con, hctx, end); + break; + #endif + #ifdef USE_BZ2LIB + case HTTP_ACCEPT_ENCODING_BZIP2: + ret = stream_bzip2_flush(srv, con, hctx, end); + break; + #endif + default: + ret = -1; + break; + } + + return ret; + } + + static int mod_deflate_stream_end(server *srv, connection *con, handler_ctx *hctx) { + int ret = -1; + switch(hctx->compression_type) { + #ifdef USE_ZLIB + case HTTP_ACCEPT_ENCODING_GZIP: + case HTTP_ACCEPT_ENCODING_DEFLATE: + ret = stream_deflate_end(srv, con, hctx); + break; + #endif + #ifdef USE_BZ2LIB + case HTTP_ACCEPT_ENCODING_BZIP2: + ret = stream_bzip2_end(srv, con, hctx); + break; + #endif + default: + ret = -1; + break; + } + + return ret; + } + + static int mod_deflate_file_chunk(server *srv, connection *con, handler_ctx *hctx, chunk *c, off_t st_size) { + plugin_data *p = hctx->plugin_data; + off_t abs_offset; + off_t toSend; + stat_cache_entry *sce = NULL; + off_t we_want_to_mmap = 2 MByte; + off_t we_want_to_send = st_size; + char *start = NULL; + + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, c->file.name, &sce)) { + log_error_write(srv, __FILE__, __LINE__, "sb", + strerror(errno), c->file.name); + return -1; + } + + abs_offset = c->file.start + c->offset; + + if (abs_offset > sce->st.st_size) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "file was shrinked:", c->file.name); + + return -1; + } + + we_want_to_send = st_size; + /* mmap the buffer + * - first mmap + * - new mmap as the we are at the end of the last one */ + if (c->file.mmap.start == MAP_FAILED || + abs_offset == (off_t)(c->file.mmap.offset + c->file.mmap.length)) { + + /* Optimizations for the future: + * + * adaptive mem-mapping + * the problem: + * we mmap() the whole file. If someone has alot large files and 32bit + * machine the virtual address area will be unrun and we will have a failing + * mmap() call. + * solution: + * only mmap 16M in one chunk and move the window as soon as we have finished + * the first 8M + * + * read-ahead buffering + * the problem: + * sending out several large files in parallel trashes the read-ahead of the + * kernel leading to long wait-for-seek times. + * solutions: (increasing complexity) + * 1. use madvise + * 2. use a internal read-ahead buffer in the chunk-structure + * 3. use non-blocking IO for file-transfers + * */ + + /* all mmap()ed areas are 512kb expect the last which might be smaller */ + size_t to_mmap; + + /* this is a remap, move the mmap-offset */ + if (c->file.mmap.start != MAP_FAILED) { + munmap(c->file.mmap.start, c->file.mmap.length); + c->file.mmap.offset += we_want_to_mmap; + } else { + /* in case the range-offset is after the first mmap()ed area we skip the area */ + c->file.mmap.offset = 0; + + while (c->file.mmap.offset + we_want_to_mmap < c->file.start) { + c->file.mmap.offset += we_want_to_mmap; + } + } + + /* length is rel, c->offset too, assume there is no limit at the mmap-boundaries */ + to_mmap = (c->file.start + c->file.length) - c->file.mmap.offset; + if(to_mmap > we_want_to_mmap) to_mmap = we_want_to_mmap; + /* we have more to send than we can mmap() at once */ + if(we_want_to_send > to_mmap) we_want_to_send = to_mmap; + + if (-1 == c->file.fd) { /* open the file if not already open */ + if (-1 == (c->file.fd = open(c->file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "sbs", "open failed for:", c->file.name, strerror(errno)); + + return -1; + } + #ifdef FD_CLOEXEC + fcntl(c->file.fd, F_SETFD, FD_CLOEXEC); + #endif + } + + if (MAP_FAILED == (c->file.mmap.start = mmap(0, to_mmap, PROT_READ, MAP_SHARED, c->file.fd, c->file.mmap.offset))) { + /* close it here, otherwise we'd have to set FD_CLOEXEC */ + + log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed:", + strerror(errno), c->file.name, c->file.fd); + + return -1; + } + + c->file.mmap.length = to_mmap; + #ifdef LOCAL_BUFFERING + buffer_copy_string_len(c->mem, c->file.mmap.start, c->file.mmap.length); + #else + #ifdef HAVE_MADVISE + /* don't advise files < 64Kb */ + if (c->file.mmap.length > (64 KByte) && + 0 != madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED)) { + log_error_write(srv, __FILE__, __LINE__, "ssbd", "madvise failed:", + strerror(errno), c->file.name, c->file.fd); + } + #endif + #endif + + /* chunk_reset() or chunk_free() will cleanup for us */ + } + + /* to_send = abs_mmap_end - abs_offset */ + toSend = (c->file.mmap.offset + c->file.mmap.length) - (abs_offset); + if(toSend > we_want_to_send) toSend = we_want_to_send; + + if (toSend < 0) { + log_error_write(srv, __FILE__, __LINE__, "soooo", + "toSend is negative:", + toSend, + c->file.mmap.length, + abs_offset, + c->file.mmap.offset); + assert(toSend < 0); + } + + #ifdef LOCAL_BUFFERING + start = c->mem->ptr; + #else + start = c->file.mmap.start; + #endif + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sdsd", + "compress file chunk: offset=", (int)c->offset, + ", toSend=", (int)toSend); + } + if (mod_deflate_compress(srv, con, hctx, + (unsigned char *)start + (abs_offset - c->file.mmap.offset), toSend) < 0) { + log_error_write(srv, __FILE__, __LINE__, "s", + "compress failed."); + return -1; + } + + c->offset += toSend; + if (c->offset == c->file.length) { + /* we don't need the mmaping anymore */ + if (c->file.mmap.start != MAP_FAILED) { + munmap(c->file.mmap.start, c->file.mmap.length); + c->file.mmap.start = MAP_FAILED; + } + } + + return toSend; + } + + static int deflate_compress_cleanup(server *srv, connection *con, handler_ctx *hctx) { + plugin_data *p = hctx->plugin_data; + int rc; + + rc = mod_deflate_stream_end(srv, con, hctx); + if(rc < 0) { + log_error_write(srv, __FILE__, __LINE__, "s", "error closing stream"); + } + + if (hctx->bytes_in < hctx->bytes_out) { + log_error_write(srv, __FILE__, __LINE__, "sbsdsd", + "uri ", con->uri.path_raw, " in=", hctx->bytes_in, " smaller than out=", hctx->bytes_out); + } + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sdsd", + " in:", hctx->bytes_in, + " out:", hctx->bytes_out); + } + + /* cleanup compression state */ + if(hctx->output != p->tmp_buf) { + buffer_free(hctx->output); + } + handler_ctx_free(hctx); + con->plugin_ctx[p->id] = NULL; + + return 0; + } + + static handler_t deflate_compress_response(server *srv, connection *con, handler_ctx *hctx, int end) { + plugin_data *p = hctx->plugin_data; + chunk *c; + size_t chunks_written = 0; + int chunk_finished = 0; + int rc=-1; + int close_stream = 0, len = 0; + unsigned int out = 0, max = 0; + + /* move all chunk from write_queue into our in_queue */ + chunkqueue_append_chunkqueue(hctx->in_queue, con->write_queue); + + len = chunkqueue_length(hctx->in_queue); + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "compress: in_queue len=", len); + } + /* calculate max bytes to compress for this call. */ + if(!end) { + max = p->conf.work_block_size * 1024; + if(max == 0 || max > len) max = len; + } else { + max = len; + } + + /* Compress chunks from in_queue into chunks for write_queue */ + for(c = hctx->in_queue->first; c && out < max; c = c->next) { + chunk_finished = 0; + len = 0; + + switch(c->type) { + case MEM_CHUNK: + len = c->mem->used - 1; + if(len > (max - out)) len = max - out; + if (mod_deflate_compress(srv, con, hctx, (unsigned char *)c->mem->ptr, len) < 0) { + log_error_write(srv, __FILE__, __LINE__, "s", + "compress failed."); + return HANDLER_ERROR; + } + c->offset += len; + out += len; + if (c->offset == c->mem->used - 1) { + chunk_finished = 1; + chunks_written++; + } + break; + case FILE_CHUNK: + len = c->file.length - c->offset; + if(len > (max - out)) len = max - out; + if ((len = mod_deflate_file_chunk(srv, con, hctx, c, len)) < 0) { + log_error_write(srv, __FILE__, __LINE__, "s", + "compress file chunk failed."); + return HANDLER_ERROR; + } + out += len; + if (c->offset == c->file.length) { + chunk_finished = 1; + chunks_written++; + } + break; + default: + + log_error_write(srv, __FILE__, __LINE__, "ds", c, "type not known"); + + return HANDLER_ERROR; + } + if(!chunk_finished) break; + } + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "compressed bytes:", out); + } + hctx->in_queue->bytes_out += out; + + if(chunks_written > 0) { + chunkqueue_remove_finished_chunks(hctx->in_queue); + } + + close_stream = (con->file_finished && chunkqueue_is_empty(hctx->in_queue)); + rc = mod_deflate_stream_flush(srv, con, hctx, close_stream); + if(rc < 0) { + log_error_write(srv, __FILE__, __LINE__, "s", "flush error"); + } + if(close_stream || end) { + deflate_compress_cleanup(srv, con, hctx); + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sbsb", + "finished uri:", con->uri.path_raw, ", query:", con->uri.query); + } + return HANDLER_FINISHED; + } else { + if(!chunkqueue_is_empty(hctx->in_queue)) { + /* We have more data to compress. */ + joblist_append(srv, con); + } + return HANDLER_COMEBACK; + } + } + + #define PATCH(x) \ + p->conf.x = s->x; + static int mod_deflate_patch_connection(server *srv, connection *con, plugin_data *p) { + size_t i, j; + plugin_config *s = p->config_storage[0]; + + PATCH(output_buffer_size); + PATCH(mimetypes); + PATCH(compression_level); + PATCH(mem_level); + PATCH(window_size); + PATCH(min_compress_size); + PATCH(work_block_size); + PATCH(enabled); + PATCH(debug); + PATCH(bzip2); + PATCH(sync_flush); + #if defined(HAVE_PCRE_H) + PATCH(nocompress_regex); + #endif + + /* skip the first, the global context */ + for (i = 1; i < srv->config_context->used; i++) { + data_config *dc = (data_config *)srv->config_context->data[i]; + s = p->config_storage[i]; + + /* condition didn't match */ + if (!config_check_cond(srv, con, dc)) continue; + + /* merge config */ + for (j = 0; j < dc->value->used; j++) { + data_unset *du = dc->value->data[j]; + + if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.output-buffer-size"))) { + PATCH(output_buffer_size); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.mimetypes"))) { + PATCH(mimetypes); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.compression-level"))) { + PATCH(compression_level); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.mem-level"))) { + PATCH(mem_level); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.window-size"))) { + PATCH(window_size); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.min-compress-size"))) { + PATCH(min_compress_size); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.work-block-size"))) { + PATCH(work_block_size); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.enabled"))) { + PATCH(enabled); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.debug"))) { + PATCH(debug); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.bzip2"))) { + PATCH(bzip2); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.sync-flush"))) { + PATCH(sync_flush); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.nocompress-url"))) { + #if defined(HAVE_PCRE_H) + PATCH(nocompress_regex); + #endif + } + } + } + + return 0; + } + #undef PATCH + + PHYSICALPATH_FUNC(mod_deflate_handle_response_start) { + plugin_data *p = p_d; + handler_ctx *hctx; + data_string *ds; + int accept_encoding = 0; + char *value; + int srv_encodings = 0; + int matched_encodings = 0; + const char *dflt_gzip = "gzip"; + const char *dflt_deflate = "deflate"; + const char *dflt_bzip2 = "bzip2"; + const char *compression_name = NULL; + int file_len=0; + int rc=-2; + int end = 0; + size_t m; + #if defined(HAVE_PCRE_H) + int n; + # define N 10 + int ovec[N * 3]; + #endif + + /* disable compression for some http status types. */ + switch(con->http_status) { + case 100: + case 101: + case 204: + case 205: + case 304: + /* disable compression as we have no response entity */ + return HANDLER_GO_ON; + default: + break; + } + + mod_deflate_patch_connection(srv, con, p); + + /* is compression allowed */ + if(!p->conf.enabled) { + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "s", "compression disabled."); + } + return HANDLER_GO_ON; + } + + #if defined(HAVE_PCRE_H) + if(p->conf.nocompress_regex) { /*check no compress regex now */ + if ((n = pcre_exec(p->conf.nocompress_regex, NULL, con->uri.path->ptr, con->uri.path->used - 1, 0, 0, ovec, 3 * N)) < 0) { + if (n != PCRE_ERROR_NOMATCH) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "execution error while matching:", n); + return HANDLER_ERROR; + } + } else { + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sb", "no compress for url:", con->uri.path); + } + return HANDLER_GO_ON; + } + } + #endif + /* Check if response has a Content-Encoding. */ + if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Content-Encoding"))) { + return HANDLER_GO_ON; + } + + /* Check Accept-Encoding for supported encoding. */ + if (NULL == (ds = (data_string *)array_get_element(con->request.headers, "Accept-Encoding"))) { + return HANDLER_GO_ON; + } + + /* get client side support encodings */ + value = ds->value->ptr; + #ifdef USE_ZLIB + if (NULL != strstr(value, "gzip")) accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; + if (NULL != strstr(value, "deflate")) accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE; + #endif + /* if (NULL != strstr(value, "compress")) accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; */ + #ifdef USE_BZ2LIB + if(p->conf.bzip2) { + if (NULL != strstr(value, "bzip2")) accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2; + } + #endif + if (NULL != strstr(value, "identity")) accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; + + /* get server side supported ones */ + #ifdef USE_BZ2LIB + if(p->conf.bzip2) { + srv_encodings |= HTTP_ACCEPT_ENCODING_BZIP2; + } + #endif + #ifdef USE_ZLIB + srv_encodings |= HTTP_ACCEPT_ENCODING_GZIP; + srv_encodings |= HTTP_ACCEPT_ENCODING_DEFLATE; + #endif + + /* find matching encodings */ + matched_encodings = accept_encoding & srv_encodings; + if (!matched_encodings) { + return HANDLER_GO_ON; + } + + if (con->request.true_http_10_client) { + /*disable gzip/bzip2 when we meet HTTP 1.0 client with Accept-Encoding + * maybe old buggy proxy server + */ + /* most of buggy clients are Yahoo Slurp;) */ + if (p->conf.debug) log_error_write(srv, __FILE__, __LINE__, "ss", + "Buggy HTTP 1.0 client sending Accept Encoding: gzip, deflate", + (char *) inet_ntop_cache_get_ip(srv, &(con->dst_addr))); + return HANDLER_GO_ON; + } + /* check if size of response is below min-compress-size */ + if(con->file_finished && con->request.http_method != HTTP_METHOD_HEAD) { + file_len = chunkqueue_length(con->write_queue); + if(file_len == 0) return HANDLER_GO_ON; + } else { + file_len = 0; + } + + if(file_len > 0 && p->conf.min_compress_size > 0 && file_len < p->conf.min_compress_size) { + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "Content-Length smaller then min_compress_size: file_len=", file_len); + } + return HANDLER_GO_ON; + } + + /* Check mimetype in response header "Content-Type" */ + if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Content-Type"))) { + int found = 0; + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "Content-Type:", ds->value); + } + for (m = 0; m < p->conf.mimetypes->used; m++) { + data_string *mimetype = (data_string *)p->conf.mimetypes->data[m]; + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "mime-type:", mimetype->value); + } + if (strncmp(mimetype->value->ptr, ds->value->ptr, mimetype->value->used-1) == 0) { + /* if (buffer_is_equal(mimetype->value, ds->value)) { */ + /* mimetype found */ + found = 1; + break; + } + } + if(!found && p->conf.mimetypes->used > 0) { + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "No compression for mimetype:", ds->value); + } + return HANDLER_GO_ON; + } + #if 0 + if(strncasecmp(ds->value->ptr, "application/x-javascript", 24) == 0) { + /*reset compress type to deflate for javascript + * prevent buggy IE6 SP1 doesn't work for js in IFrame + */ + matched_encodings = HTTP_ACCEPT_ENCODING_DEFLATE; + } + #endif + } + + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "enable compression for ", con->uri.path); + } + + /* the response might change according to Accept-Encoding */ + if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Vary"))) { + /* append Accept-Encoding to Vary header */ + if (NULL == strstr(ds->value->ptr, "Accept-Encoding")) { + buffer_append_string(ds->value, ",Accept-Encoding"); + if (p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "appending ,Accept-Encoding for ", con->uri.path); + } + } + } else { + if (p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "add Vary: Accept-Encoding for ", con->uri.path); + } + response_header_insert(srv, con, CONST_STR_LEN("Vary"), + CONST_STR_LEN("Accept-Encoding")); + } + + /* enable compression */ + hctx = handler_ctx_init(); + hctx->plugin_data = p; + + /* select best matching encoding */ + if (matched_encodings & HTTP_ACCEPT_ENCODING_BZIP2) { + hctx->compression_type = HTTP_ACCEPT_ENCODING_BZIP2; + compression_name = dflt_bzip2; + rc = stream_bzip2_init(srv, con, hctx); + } else if (matched_encodings & HTTP_ACCEPT_ENCODING_GZIP) { + hctx->compression_type = HTTP_ACCEPT_ENCODING_GZIP; + compression_name = dflt_gzip; + rc = stream_deflate_init(srv, con, hctx); + } else if (matched_encodings & HTTP_ACCEPT_ENCODING_DEFLATE) { + hctx->compression_type = HTTP_ACCEPT_ENCODING_DEFLATE; + compression_name = dflt_deflate; + rc = stream_deflate_init(srv, con, hctx); + } + if(rc == -1) { + log_error_write(srv, __FILE__, __LINE__, "s", + "Failed to initialize compression."); + } + + if(rc < 0) { + handler_ctx_free(hctx); + return HANDLER_GO_ON; + } + + /* setup output buffer. */ + if(p->conf.sync_flush || p->conf.output_buffer_size == 0) { + buffer_prepare_copy(p->tmp_buf, 32 * 1024); + hctx->output = p->tmp_buf; + } else { + hctx->output = buffer_init(); + buffer_prepare_copy(hctx->output, p->conf.output_buffer_size); + } + con->plugin_ctx[p->id] = hctx; + + /* set Content-Encoding to show selected compression type. */ + response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name)); + + if (con->file_finished) end = 1; + + con->parsed_response &= ~(HTTP_CONTENT_LENGTH); + #if 0 + /* debug */ + if (con->parsed_response & HTTP_TRANSFER_ENCODING_CHUNKED) + log_error_write(srv, __FILE__, __LINE__, "s", + "deflate: response with chunked encoding"); + if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) + log_error_write(srv, __FILE__, __LINE__, "s", + "deflate: transfer encoding with chunked encoding"); + #endif + + if (con->file_finished && (p->conf.work_block_size == 0 || file_len < (p->conf.work_block_size * 1024)) + && con->request.http_method != HTTP_METHOD_HEAD) { + /* disable chunk transfer */ + con->response.transfer_encoding = 0; + con->parsed_response &= ~(HTTP_TRANSFER_ENCODING_CHUNKED); + if(p->conf.debug) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "Compress all content and use Content-Length header: uncompress len=", file_len); + } + return deflate_compress_response(srv, con, hctx, end); + } else { + if (con->request.http_version == HTTP_VERSION_1_1) { + if (p->conf.debug) + log_error_write(srv, __FILE__, __LINE__, "sb", + "chunk transfer encoding for uri", con->uri.path); + /* Make sure to use chunked encoding. */ + con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED; + } else { + if (p->conf.debug) + log_error_write(srv, __FILE__, __LINE__, "sb", + "http 1.0 encoding for uri", con->uri.path); + /* We don't have to use chunked encoding because HTTP 1.0 don't support it. */ + con->response.transfer_encoding = 0; + con->parsed_response &= ~(HTTP_TRANSFER_ENCODING_CHUNKED); + } + } + + if (p->conf.debug) + log_error_write(srv, __FILE__, __LINE__, "sdsb", "end =", end, "for uri", con->uri.path); + + deflate_compress_response(srv, con, hctx, end); + return HANDLER_GO_ON; + } + + JOBLIST_FUNC(mod_deflate_handle_response_filter) { + plugin_data *p = p_d; + handler_ctx *hctx = con->plugin_ctx[p->id]; + + if(hctx == NULL) return HANDLER_GO_ON; + if(!hctx->stream_open) return HANDLER_GO_ON; + if(con->request.http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON; + + return deflate_compress_response(srv, con, hctx, 0); + } + + handler_t mod_deflate_cleanup(server *srv, connection *con, void *p_d) { + plugin_data *p = p_d; + handler_ctx *hctx = con->plugin_ctx[p->id]; + + if(hctx == NULL) return HANDLER_GO_ON; + + if(p->conf.debug && hctx->stream_open) { + log_error_write(srv, __FILE__, __LINE__, "sbsb", + "stream open at cleanup. uri=", con->uri.path_raw, ", query=", con->uri.query); + } + + deflate_compress_cleanup(srv, con, hctx); + + return HANDLER_GO_ON; + } + + int mod_deflate_plugin_init(plugin *p) { + p->version = LIGHTTPD_VERSION_ID; + p->name = buffer_init_string("deflate"); + + p->init = mod_deflate_init; + p->cleanup = mod_deflate_free; + p->set_defaults = mod_deflate_setdefaults; + p->connection_reset = mod_deflate_cleanup; + p->handle_connection_close = mod_deflate_cleanup; + p->handle_response_start = mod_deflate_handle_response_start; + p->handle_response_filter = mod_deflate_handle_response_filter; + + p->data = NULL; + + return 0; + } *** ../lighttpd-1.4.19/src/plugin.c 2006-10-09 21:50:29.000000000 +0800 --- src/plugin.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 38,47 **** --- 38,49 ---- PLUGIN_FUNC_HANDLE_CONNECTION_CLOSE, PLUGIN_FUNC_HANDLE_TRIGGER, PLUGIN_FUNC_HANDLE_SIGHUP, PLUGIN_FUNC_HANDLE_SUBREQUEST, PLUGIN_FUNC_HANDLE_SUBREQUEST_START, + PLUGIN_FUNC_HANDLE_RESPONSE_START, + PLUGIN_FUNC_HANDLE_RESPONSE_FILTER, PLUGIN_FUNC_HANDLE_JOBLIST, PLUGIN_FUNC_HANDLE_DOCROOT, PLUGIN_FUNC_HANDLE_PHYSICAL, PLUGIN_FUNC_CONNECTION_RESET, PLUGIN_FUNC_INIT, *************** *** 264,273 **** --- 266,277 ---- PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_URI_RAW, handle_uri_raw) PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_REQUEST_DONE, handle_request_done) PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_CONNECTION_CLOSE, handle_connection_close) PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SUBREQUEST, handle_subrequest) PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SUBREQUEST_START, handle_subrequest_start) + PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_START, handle_response_start) + PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_FILTER, handle_response_filter) PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_JOBLIST, handle_joblist) PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_DOCROOT, handle_docroot) PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_PHYSICAL, handle_physical) PLUGIN_TO_SLOT(PLUGIN_FUNC_CONNECTION_RESET, connection_reset) *************** *** 393,402 **** --- 397,408 ---- PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_CONNECTION_CLOSE, handle_connection_close); PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_TRIGGER, handle_trigger); PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SIGHUP, handle_sighup); PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SUBREQUEST, handle_subrequest); PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SUBREQUEST_START, handle_subrequest_start); + PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_START, handle_response_start); + PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_FILTER, handle_response_filter); PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_JOBLIST, handle_joblist); PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_DOCROOT, handle_docroot); PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_PHYSICAL, handle_physical); PLUGIN_TO_SLOT(PLUGIN_FUNC_CONNECTION_RESET, connection_reset); PLUGIN_TO_SLOT(PLUGIN_FUNC_CLEANUP, cleanup); *** ../lighttpd-1.4.19/src/plugin.h 2006-10-04 21:26:23.000000000 +0800 --- src/plugin.h 2008-09-17 14:42:22.000000000 +0800 *************** *** 52,61 **** --- 52,63 ---- /* when a handler for the request * has to be found */ handler_t (* handle_subrequest) (server *srv, connection *con, void *p_d); /* */ + handler_t (* handle_response_start) (server *srv, connection *con, void *p_d); /* before response headers are written */ + handler_t (* handle_response_filter) (server *srv, connection *con, void *p_d); /* response content filter */ handler_t (* connection_reset) (server *srv, connection *con, void *p_d); /* */ void *data; /* dlopen handle */ void *lib; *************** *** 66,75 **** --- 68,79 ---- handler_t plugins_call_handle_uri_raw(server *srv, connection *con); handler_t plugins_call_handle_uri_clean(server *srv, connection *con); handler_t plugins_call_handle_subrequest_start(server *srv, connection *con); handler_t plugins_call_handle_subrequest(server *srv, connection *con); + handler_t plugins_call_handle_response_start(server *srv, connection *con); + handler_t plugins_call_handle_response_filter(server *srv, connection *con); handler_t plugins_call_handle_request_done(server *srv, connection *con); handler_t plugins_call_handle_docroot(server *srv, connection *con); handler_t plugins_call_handle_physical(server *srv, connection *con); handler_t plugins_call_handle_connection_close(server *srv, connection *con); handler_t plugins_call_handle_joblist(server *srv, connection *con); *** ../lighttpd-1.4.19/src/request.c 2007-08-18 19:14:12.000000000 +0800 --- src/request.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 413,424 **** --- 413,426 ---- return 0; } if (major_num == 1 && minor_num == 1) { con->request.http_version = con->conf.allow_http11 ? HTTP_VERSION_1_1 : HTTP_VERSION_1_0; + con->request.true_http_10_client = 0; } else if (major_num == 1 && minor_num == 0) { con->request.http_version = HTTP_VERSION_1_0; + con->request.true_http_10_client = 1; } else { con->http_status = 505; if (srv->srvconf.log_request_header_on_error) { log_error_write(srv, __FILE__, __LINE__, "s", "unknown HTTP version -> 505"); *** ../lighttpd-1.4.19/src/response.c 2008-03-02 20:59:18.000000000 +0800 --- src/response.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 30,40 **** buffer *b; size_t i; int have_date = 0; int have_server = 0; ! b = chunkqueue_get_prepend_buffer(con->write_queue); if (con->request.http_version == HTTP_VERSION_1_1) { BUFFER_COPY_STRING_CONST(b, "HTTP/1.1 "); } else { BUFFER_COPY_STRING_CONST(b, "HTTP/1.0 "); --- 30,40 ---- buffer *b; size_t i; int have_date = 0; int have_server = 0; ! b = chunkqueue_get_prepend_buffer(con->output_queue); if (con->request.http_version == HTTP_VERSION_1_1) { BUFFER_COPY_STRING_CONST(b, "HTTP/1.1 "); } else { BUFFER_COPY_STRING_CONST(b, "HTTP/1.0 "); *************** *** 47,56 **** --- 47,58 ---- BUFFER_APPEND_STRING_CONST(b, "\r\nConnection: "); buffer_append_string(b, con->keep_alive ? "keep-alive" : "close"); } if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) { + /* if (!(con->parsed_response & HTTP_TRANSFER_ENCODING_CHUNKED) && + (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED)) { */ BUFFER_APPEND_STRING_CONST(b, "\r\nTransfer-Encoding: chunked"); } /* add all headers */ *************** *** 63,72 **** --- 65,84 ---- 0 != strncmp(ds->key->ptr, "X-LIGHTTPD-", sizeof("X-LIGHTTPD-") - 1) && 0 != strncmp(ds->key->ptr, "X-Sendfile", sizeof("X-Sendfile") - 1)) { if (buffer_is_equal_string(ds->key, CONST_STR_LEN("Date"))) have_date = 1; if (buffer_is_equal_string(ds->key, CONST_STR_LEN("Server"))) have_server = 1; + /* remove Transfer-Encoding: chunked header when HTTP 1.0 + * or transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED == 0 + */ + if ( (con->request.http_version == HTTP_VERSION_1_0 || + !(con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED)) && + 0 == strncasecmp(ds->value->ptr, "chunked", sizeof("chunked")-1) && + 0 == strncasecmp(ds->key->ptr, "Transfer-Encoding", sizeof("Transfer-Encoding") - 1)) { + continue ; + } + BUFFER_APPEND_STRING_CONST(b, "\r\n"); buffer_append_string_buffer(b, ds->key); BUFFER_APPEND_STRING_CONST(b, ": "); #if 0 /** *************** *** 574,584 **** /* restore '/' */ *pathinfo = '/'; } if (slash) pathinfo = slash; ! } while ((found == 0) && (slash != NULL) && ((size_t)(slash - srv->tmp_buf->ptr) > (con->physical.basedir->used - 2))); if (found == 0) { /* no it really doesn't exists */ con->http_status = 404; --- 586,596 ---- /* restore '/' */ *pathinfo = '/'; } if (slash) pathinfo = slash; ! } while ((found == 0) && (slash != NULL) && ((size_t)(slash - srv->tmp_buf->ptr) > con->physical.basedir->used - 2)); if (found == 0) { /* no it really doesn't exists */ con->http_status = 404; *** ../lighttpd-1.4.19/src/server.c 2008-02-27 00:20:04.000000000 +0800 --- src/server.c 2008-09-17 14:42:22.000000000 +0800 *************** *** 201,210 **** --- 201,213 ---- assert(srv->conns); srv->joblist = calloc(1, sizeof(*srv->joblist)); assert(srv->joblist); + srv->joblist_prev = calloc(1, sizeof(*srv->joblist)); + assert(srv->joblist_prev); + srv->fdwaitqueue = calloc(1, sizeof(*srv->fdwaitqueue)); assert(srv->fdwaitqueue); srv->srvconf.modules = array_init(); srv->srvconf.modules_dir = buffer_init_string(LIBRARY_DIR); *************** *** 292,301 **** --- 295,305 ---- CLEAN(status); CLEAN(srvconf.upload_tempdirs); #undef CLEAN joblist_free(srv, srv->joblist); + joblist_free(srv, srv->joblist_prev); fdwaitqueue_free(srv, srv->fdwaitqueue); if (srv->stat_cache) { stat_cache_free(srv->stat_cache); } *************** *** 1135,1144 **** --- 1139,1149 ---- } /* main-loop */ while (!srv_shutdown) { int n; + int timeout; size_t ndx; time_t min_ts; if (handle_sig_hup) { handler_t r; *************** *** 1386,1396 **** srv->want_fds--; } } ! if ((n = fdevent_poll(srv->ev, 1000)) > 0) { /* n is the number of events */ int revents; int fd_ndx; #if 0 if (n > 0) { --- 1391,1406 ---- srv->want_fds--; } } ! if(srv->joblist->used > 0) { ! timeout = 500; ! } else { ! timeout = 1000; ! } ! if ((n = fdevent_poll(srv->ev, timeout)) > 0) { /* n is the number of events */ int revents; int fd_ndx; #if 0 if (n > 0) { *************** *** 1434,1462 **** log_error_write(srv, __FILE__, __LINE__, "ss", "fdevent_poll failed:", strerror(errno)); } ! for (ndx = 0; ndx < srv->joblist->used; ndx++) { ! connection *con = srv->joblist->ptr[ndx]; handler_t r; connection_state_machine(srv, con); switch(r = plugins_call_handle_joblist(srv, con)) { case HANDLER_FINISHED: case HANDLER_GO_ON: break; default: log_error_write(srv, __FILE__, __LINE__, "d", r); break; } - - con->in_joblist = 0; } ! ! srv->joblist->used = 0; } if (srv->srvconf.pid_file->used && srv->srvconf.changeroot->used == 0 && 0 == graceful_shutdown) { --- 1444,1476 ---- log_error_write(srv, __FILE__, __LINE__, "ss", "fdevent_poll failed:", strerror(errno)); } ! if(srv->joblist->used > 0) { ! connections *joblist = srv->joblist; ! /* switch joblist queues. */ ! srv->joblist = srv->joblist_prev; ! srv->joblist_prev = joblist; ! for (ndx = 0; ndx < joblist->used; ndx++) { ! connection *con = joblist->ptr[ndx]; handler_t r; + con->in_joblist = 0; connection_state_machine(srv, con); switch(r = plugins_call_handle_joblist(srv, con)) { case HANDLER_FINISHED: case HANDLER_GO_ON: break; default: log_error_write(srv, __FILE__, __LINE__, "d", r); break; } } ! joblist->used = 0; ! } } if (srv->srvconf.pid_file->used && srv->srvconf.changeroot->used == 0 && 0 == graceful_shutdown) { >Release-Note: >Audit-Trail: Responsible-Changed-From-To: freebsd-bugs->mnag Responsible-Changed-By: linimon Responsible-Changed-When: Wed Sep 17 08:23:33 UTC 2008 Responsible-Changed-Why: Make this a ports PR and assign. http://www.freebsd.org/cgi/query-pr.cgi?pr=127435 State-Changed-From-To: open->closed State-Changed-By: mnag State-Changed-When: Sun Jun 28 22:17:55 UTC 2009 State-Changed-Why: It's rejected by upstream lighttpd maintainers. Seems that will be included in 1.5 version. Big features need to be incorporated first by upstream maintainers. Thanks. http://www.freebsd.org/cgi/query-pr.cgi?pr=127435 >Unformatted: