Re: [tproxy] [HELP] Tproxy server Can't receive any client packet

[linuxpark <linuxpark@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 3972 bytes --]

Hi

i have one more problem and question :-(
i tested tproxy4 in debian linux and succeed in  transparent proxying 
function of the apache proxy server (a tproxy patched)

but when i tested the same apache version in  Cento  OS (5.4) ,

the apache proxy did not operate correctly. listen () function did not 
return when a client tried to connect a web server.
then the apache did not do any action. no response but the usage count 
of filter in the mangle table was increased.

and then.. i found something strange like below. there are two listening 
daemon with same port but the daemons are same process


root@waf:~/tproxy4-20091127/httpd-2.2.9-tproxy-v1# netstat -ap | grep http
tcp        0      0 *:squid                     
*:*                         LISTEN      7397/httpd         tcp        
0      0 *:squid                     *:*                         
LISTEN      7397/httpd


i don't know why the apache proxy which succeed in doing tproxy in the 
debian linux did not operate correctly in CentOS (5.4)
but squid proxy succeed in tproxy  in both the debian and the CentOS box

do you have any idea ?  previously thanks ~

Cheers,
JeHo Park 



[Environment]

- bridged  network

Client <---> [eth2   apache tproxy   eth1]  <---> WebServer
                       +--------    br0  ---------+

- CPU: AMD Phenom(tm) II X4 955 Processor

- OS:
 CentOS release 5.4 (Final)

- Kernel:  vannilla 2.6.31

- Kernel config:
 kconfig-2.6.31  [attached]

- apache server:
 apache 2.2.9 +  http-2.2.9-tproxy4.patch [attached]

- iptables:
 1.4.3

- gcc
 4.1.2

*- iptables & ebroute & ip rules*

   ${IPTABLES} -t mangle -F
   ${IPTABLES} -t mangle -X DIVERT
   ${IPTABLES} -t nat -F
     #ip rule del fwmark 0x1/0x1 lookup 100
   ip rule del fwmark 1 lookup 100
   ip route del local 0.0.0.0/0 dev lo table 100
   ebtables -t broute -F

   ${IPTABLES} -t mangle -N DIVERT
   ${IPTABLES} -t mangle -A PREROUTING -p tcp -m socket -j DIVERT
   ${IPTABLES} -t mangle -A DIVERT -j MARK --set-mark 1
   ${IPTABLES} -t mangle -A DIVERT -j ACCEPT
   ${IPTABLES} -t mangle -A PREROUTING -p tcp --dport 80 -j TPROXY 
--tproxy-mark 0x1/0x1 --on-port ${PORT}

   ip rule add fwmark 1 lookup 100
   ip route add local 0.0.0.0/0 dev lo table 100

   ebtables -t broute -A BROUTING -i $CLIENT_IFACE -p ipv4 --ip-proto 
tcp --ip-dport 80 -j redirect --redirect-target ACCEPT
   ebtables -t broute -A BROUTING -i $INET_IFACE -p ipv4 --ip-proto tcp 
--ip-sport 80 -j redirect --redirect-target ACCEPT


   cd /proc/sys/net/bridge/
   for i in *
       do
           echo 0 > $i
       done
       unset i

   sysctl net.ipv4.ip_nonlocal_bind=1
   sysctl net.ipv4.ip_forward=1



KOVACS Krisztian 쓴 글:
> Hi,
>
> On Mon, 2009-11-23 at 15:51 +0900, 박제호 wrote:
>   
>> i have a problem in my transparent proxy test,
>> i recently made up the testbed as below to run the tproxy patched
>> apache proxy [mod_proxy],
>> and i applied all iptables and routing rules with referencing the
>> readme file [http://www.balabit.com/downloads/files/tproxy/README.txt,
>> http://www.mjmwired.net/kernel/Documentation/networking/tproxy.txt]
>> the proxy server listening the port 3128 and i checked there were no problem.
>> but  when the client tried to connect the web server,
>> the packets reached to the box and i found the usage counts of filter
>> rules in the mangle table incresed
>> but my tproxy server could not receive any corresponding packet from the socket
>>
>> I want to know why my proxy server can't receive any packet through the socket,
>> Do i need some more DNAT rules ?
>>     
>
> Would you mind testing the setup with an unpatched upstream kernel, that
> has tproxy built-in? (2.6.31, for example)
>
> Also, please download the latest iptables from netfilter.org and try
> using that. (No need for patching, tproxy support is in upstream.)
>
> That would help a lot in identifying the source of the issue. Thanks in
> advance.
>
> Cheers,
> Krisztian
>
>
>   


[-- Attachment #2: http-2.2.9-tproxy4-patch --]
[-- Type: text/plain, Size: 12370 bytes --]

diff -uNr httpd-2.2.9/modules/proxy/mod_proxy.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.c
--- httpd-2.2.9/modules/proxy/mod_proxy.c	2008-05-29 06:11:24.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.c	2009-12-02 11:26:25.000000000 +0900
@@ -1870,6 +1870,18 @@
     return NULL;
 }
 
+static const char *
+set_tproxy(cmd_parms *parms, void *dummy, int flag)
+{
+    proxy_server_conf *psf =
+	ap_get_module_config(parms->server->module_config, &proxy_module);
+
+    psf->tproxy = flag;
+    psf->tproxy_set = 1;
+
+    return NULL;
+}
+
 static void ap_add_per_proxy_conf(server_rec *s, ap_conf_vector_t *dir_config)
 {
     proxy_server_conf *sconf = ap_get_module_config(s->module_config,
@@ -2082,6 +2094,8 @@
      "A balancer or worker name with list of params"),
     AP_INIT_TAKE1("ProxyFtpDirCharset", set_ftp_directory_charset, NULL,
      RSRC_CONF|ACCESS_CONF, "Define the character set for proxied FTP listings"),
+    AP_INIT_FLAG("ProxyTProxy", set_tproxy, NULL, RSRC_CONF,
+     "on if the proxy should use client IP to connect to the origin server"),
     {NULL}
 };
 
diff -uNr httpd-2.2.9/modules/proxy/mod_proxy.h httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.h
--- httpd-2.2.9/modules/proxy/mod_proxy.h	2008-05-29 06:24:04.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.h	2009-12-02 11:26:25.000000000 +0900
@@ -193,6 +193,9 @@
     } proxy_status;             /* Status display options */
     char proxy_status_set;
     apr_pool_t *pool;           /* Pool used for allocating this struct */
+    int tproxy;                 /* true if tproxy is enabled */
+    char tproxy_set;
+
 } proxy_server_conf;
 
 
@@ -711,6 +714,7 @@
  * @param conn    acquired connection
  * @param worker  connection worker
  * @param s       current server record
+ * @param remote_addr  client addr
  * @return        OK or HTTP_XXX error
  * @note In case the socket already exists for conn, just check the link
  * status.
@@ -718,7 +722,8 @@
 PROXY_DECLARE(int) ap_proxy_connect_backend(const char *proxy_function,
                                             proxy_conn_rec *conn,
                                             proxy_worker *worker,
-                                            server_rec *s);
+                                            server_rec *s,
+					    apr_sockaddr_t *remote_addr);
 /**
  * Make a connection record for backend connection
  * @param proxy_function calling proxy scheme (http, ajp, ...)
diff -uNr httpd-2.2.9/modules/proxy/mod_proxy_ajp.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ajp.c
--- httpd-2.2.9/modules/proxy/mod_proxy_ajp.c	2008-06-05 21:46:43.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ajp.c	2009-12-02 11:26:25.000000000 +0900
@@ -605,11 +605,11 @@
         goto cleanup;
 
     /* Step Two: Make the Connection */
-    if (ap_proxy_connect_backend(scheme, backend, worker, r->server)) {
-        ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
-                     "proxy: AJP: failed to make connection to backend: %s",
-                     backend->hostname);
-        status = HTTP_SERVICE_UNAVAILABLE;
+    if (ap_proxy_connect_backend(scheme, backend, worker, r->server, r->connection->remote_addr)) {
+	ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
+		     "proxy: AJP: failed to make connection to backend: %s",
+		     backend->hostname);
+	status = HTTP_SERVICE_UNAVAILABLE;
         goto cleanup;
     }
 
diff -uNr httpd-2.2.9/modules/proxy/mod_proxy_ftp.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ftp.c
--- httpd-2.2.9/modules/proxy/mod_proxy_ftp.c	2008-05-18 04:42:03.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ftp.c	2009-12-02 11:26:25.000000000 +0900
@@ -973,11 +973,11 @@
      */
 
 
-    if (ap_proxy_connect_backend("FTP", backend, worker, r->server)) {
-        ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
-                     "proxy: FTP: an error occurred creating a new connection to %pI (%s)",
-                     connect_addr, connectname);
-        proxy_ftp_cleanup(r, backend);
+    if (ap_proxy_connect_backend("FTP", backend, worker, r->server, c->remote_addr)) {
+	ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
+		     "proxy: FTP: an error occurred creating a new connection to %pI (%s)",
+		     connect_addr, connectname);
+	proxy_ftp_cleanup(r, backend);
         return HTTP_SERVICE_UNAVAILABLE;
     }
 
diff -uNr httpd-2.2.9/modules/proxy/mod_proxy_http.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_http.c
--- httpd-2.2.9/modules/proxy/mod_proxy_http.c	2008-06-11 03:49:19.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_http.c	2009-12-02 11:26:25.000000000 +0900
@@ -1871,6 +1871,10 @@
      * connection ID of the current upstream connection is the same as that
      * of the connection when the socket was opened.
      */
+        
+    ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
+                     "--> linuxpark:  %s/%s/%d %s\n", 
+		     __FILE__, __FUNCTION__, __LINE__, url);
     apr_pool_t *p = r->connection->pool;
     conn_rec *c = r->connection;
     apr_uri_t *uri = apr_palloc(r->connection->pool, sizeof(*uri));
@@ -1929,7 +1933,7 @@
         goto cleanup;
 
     /* Step Two: Make the Connection */
-    if (ap_proxy_connect_backend(proxy_function, backend, worker, r->server)) {
+     if (ap_proxy_connect_backend(proxy_function, backend, worker, r->server, c->remote_addr)) {
         if (r->proxyreq == PROXYREQ_PROXY)
             status = HTTP_NOT_FOUND;
         else
diff -uNr httpd-2.2.9/modules/proxy/proxy_util.c httpd-2.2.9-tproxy-v1/modules/proxy/proxy_util.c
--- httpd-2.2.9/modules/proxy/proxy_util.c	2008-05-29 06:11:24.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/modules/proxy/proxy_util.c	2009-12-02 11:26:25.000000000 +0900
@@ -1596,8 +1596,8 @@
         }
 
         ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,
-                     "proxy: %s: fam %d socket created to connect to %s",
-                     proxy_function, backend_addr->family, backend_name);
+                     "%s/%d proxy: %s: fam %d socket created to connect to %s",
+                     __FUNCTION__, __LINE__, proxy_function, backend_addr->family, backend_name);
 
         /* make the connection out of the socket */
         rv = apr_socket_connect(*newsock, backend_addr);
@@ -2269,7 +2269,8 @@
 PROXY_DECLARE(int) ap_proxy_connect_backend(const char *proxy_function,
                                             proxy_conn_rec *conn,
                                             proxy_worker *worker,
-                                            server_rec *s)
+                                            server_rec *s,
+					    apr_sockaddr_t *remote_addr)
 {
     apr_status_t rv;
     int connected = 0;
@@ -2342,14 +2343,31 @@
                 ap_log_error(APLOG_MARK, APLOG_ERR, rv, s,
                              "apr_socket_opt_set(SO_KEEPALIVE): Failed to set"
                              " Keepalive");
-            }
-        }
+	    }
+	}
+
+	/* TProxy support */
+	if (conf->tproxy) {
+#if 1 
+	    //if ((rv = apr_socket_opt_set(newsock, APR_IP_TRANSPARENT, 1)) != APR_SUCCESS) {
+	    if ((rv = apr_socket_opt_set(newsock, 32, 1)) != APR_SUCCESS) {
+		ap_log_error(APLOG_MARK, APLOG_ERR, rv, s,
+			     "apr_socket_opt_set(APR_IP_TRANSPARENT): Failed to set");
+	    }
+#endif
+	    if ((rv = apr_socket_bind(newsock, remote_addr)) != APR_SUCCESS) {
+		ap_log_error(APLOG_MARK, APLOG_ERR, rv, s,
+			     "proxy: error binding to socket %pI", remote_addr);
+		return HTTP_INTERNAL_SERVER_ERROR;
+	    }
+	}
+
         ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,
-                     "proxy: %s: fam %d socket created to connect to %s",
-                     proxy_function, backend_addr->family, worker->hostname);
+                     "%s/%d proxy: %s: fam %d socket created to connect to %s",
+                     __FUNCTION__, __LINE__, proxy_function, backend_addr->family, worker->hostname);
 
-        /* make the connection out of the socket */
-        rv = apr_socket_connect(newsock, backend_addr);
+	/* make the connection out of the socket */
+	rv = apr_socket_connect(newsock, backend_addr);
 
         /* if an error occurred, loop round and try again */
         if (rv != APR_SUCCESS) {
diff -uNr httpd-2.2.9/os/unix/unixd.c httpd-2.2.9-tproxy-v1/os/unix/unixd.c
--- httpd-2.2.9/os/unix/unixd.c	2006-07-12 12:38:44.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/os/unix/unixd.c	2009-12-02 11:26:26.000000000 +0900
@@ -49,8 +49,10 @@
 #endif
 #ifdef HAVE_SYS_PRCTL_H
 #include <sys/prctl.h>
+#include <sys/capability.h>
 #endif
 
+
 unixd_config_rec unixd_config;
 
 /* Set group privileges.
@@ -60,6 +62,36 @@
  * with different sets of groups for each.
  */
 
+void
+keep_capabilities(void)
+{
+    if (prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) != 0) {
+	ap_log_error(APLOG_MARK, APLOG_ERR, errno, NULL, "prctl Failed");
+    }
+}
+
+static void
+set_capabilities(int capability)
+{
+    cap_user_header_t head = (cap_user_header_t) calloc(1, sizeof(*head));
+    cap_user_data_t cap = (cap_user_data_t) calloc(1, sizeof(*cap));
+
+    head->version = _LINUX_CAPABILITY_VERSION;
+    head->pid = 0;
+    cap->effective = (1 << capability);
+    cap->inheritable = cap->effective;
+    cap->permitted = cap->effective;
+
+    if (capset(head, cap) != 0) {
+	ap_log_error(APLOG_MARK, APLOG_ERR, errno, NULL, "capset Failed");
+    }
+
+    free(head);
+    free(cap);
+}
+
+
+
 static int set_group_privs(void)
 {
     if (!geteuid()) {
@@ -117,6 +149,7 @@
     if (set_group_privs()) {
         return -1;
     }
+    keep_capabilities();
 #ifdef MPE
     /* Only try to switch if we're running as MANAGER.SYS */
     if (geteuid() == 1 && unixd_config.user_id > 1) {
@@ -142,6 +175,10 @@
                     (long) unixd_config.user_id);
         return -1;
     }
+
+    /* Needed for tproxy */
+    set_capabilities(CAP_NET_ADMIN);
+
 #if defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE)
     /* this applies to Linux 2.4+ */
 #ifdef AP_MPM_WANT_SET_COREDUMPDIR
@@ -461,7 +498,6 @@
 #ifdef _OSD_POSIX
     int sockdes;
 #endif
-
     *accepted = NULL;
     status = apr_socket_accept(&csd, lr->sd, ptrans);
     if (status == APR_SUCCESS) {
diff -uNr httpd-2.2.9/srclib/apr/include/apr_network_io.h httpd-2.2.9-tproxy-v1/srclib/apr/include/apr_network_io.h
--- httpd-2.2.9/srclib/apr/include/apr_network_io.h	2007-12-15 13:10:38.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/srclib/apr/include/apr_network_io.h	2009-12-02 11:26:30.000000000 +0900
@@ -65,6 +65,7 @@
 #define APR_SO_DEBUG         4    /**< Debug */
 #define APR_SO_NONBLOCK      8    /**< Non-blocking IO */
 #define APR_SO_REUSEADDR     16   /**< Reuse addresses */
+#define APR_IP_TRANSPARENT   32   /**< TProxy support */
 #define APR_SO_SNDBUF        64   /**< Send buffer */
 #define APR_SO_RCVBUF        128  /**< Receive buffer */
 #define APR_SO_DISCONNECTED  256  /**< Disconnected */
diff -uNr httpd-2.2.9/srclib/apr/network_io/unix/sockets.c httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockets.c
--- httpd-2.2.9/srclib/apr/network_io/unix/sockets.c	2009-12-02 14:18:45.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockets.c	2009-12-02 11:26:30.000000000 +0900
@@ -151,6 +151,9 @@
 
 apr_status_t apr_socket_bind(apr_socket_t *sock, apr_sockaddr_t *sa)
 {
+    int val = 1;
+
+    setsockopt (sock->socketdes, SOL_IP, 19, &val, sizeof (val));
     if (bind(sock->socketdes, 
              (struct sockaddr *)&sa->sa, sa->salen) == -1) {
         return errno;
     }
diff -uNr httpd-2.2.9/srclib/apr/network_io/unix/sockopt.c httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockopt.c
--- httpd-2.2.9/srclib/apr/network_io/unix/sockopt.c	2006-08-03 19:55:31.000000000 +0900
+++ httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockopt.c	2009-12-02 11:26:30.000000000 +0900
@@ -17,6 +17,7 @@
 #include "apr_arch_networkio.h"
 #include "apr_strings.h"
 
+#define IP_TRANSPARENT 19
 
 static apr_status_t soblock(int sd)
 {
@@ -318,6 +319,15 @@
         return APR_ENOTIMPL;
 #endif
         break;
+    case APR_IP_TRANSPARENT:
+#ifdef IP_TRANSPARENT
+	if (setsockopt(sock->socketdes, SOL_IP, IP_TRANSPARENT, (void *)&one, sizeof(int)) == -1) {
+	    return errno;
+	}
+#else
+	return APR_ENOTIMPL;
+#endif
+	break;
     default:
         return APR_EINVAL;
     }