Merge branch 'v5.3.dev' into v5.3

Conflicts resolved:
	.gitlab-ci.yml
	ChangeLog

Author: philipianpearce

.gitlab-ci.yml

@@ -1,8 +1,23 @@
 About "Bug" tags show "https://github.com/e2guardian/e2guardian/issues?q=is%3Aissue+is%3Aclosed" 
 
+version 5.3.5
+
+February 2020 to August 2020
+
+- Fix bug #619 x-forwarded wrong IP in MITM requests
+- Fix #607 - exceptionmimetypes not working
+- Fix #592 - last line of config file(s) not being read
+- Fix #590 - Storyboard line not being parsed when trailing comment present
+- Fix #585 - based on bmuel suggestions on by-pass
+- Fix #520 - bypass cookie generation and checks do not match
+
+- Add note re amending site.story for virus exceptions #577
+- Update configure.ac for raspbery pi
+
 version 5.3.4
 
 January 2020
+
 - Increase example maxcontentcachesize to make filtering youtube work
 - Fix #565 segfault when no write permission on generated certs directory 
 - Fix #493 referexception not working

ChangeLog

@@ -1,6 +1,6 @@
 # [E2Guardian](http://e2guardian.org) 
 
-This is the v5 production version - v5.3.4 
+This is the final maintenance release for v5.3 production version - v5.3.5
 
 Note that large sections of the code has been re-written and there are
 significant changes to the configuration files in this release.
@@ -27,8 +27,8 @@ Read the INSTALL for installation instructions.
 
 Github: https://github.com/e2guardian
 
-*	v5develop is used to develop new features
-*	Bugfixes primarily occurs in the version branch 
+*	current development version i.e. v5.5.dev is used to develop new features
+*	Bug fixes primarily occurs in the version '.dev' branch
 
 ## Bugs and Feature Requests
 

README.md

@@ -114,7 +114,7 @@ sitelist = 'name=localexception,messageno=662,path=@DGCONFDIR@/lists/localexcept
 # Uncomment the two lines below if want to only allow extentions/mime types in these lists
 # You will also need to uncomment the checkfiletype function in site.story to enable this
 #fileextlist = 'name=exceptionextension,path=@DGCONFDIR@/lists/exceptionextensionlist'
-#mimelist = 'name=exceptionmime,path=@DGCONFDIR@/lists/exceptionmimelist'
+#mimelist = 'name=exceptionmime,path=@DGCONFDIR@/lists/exceptionmimetypelist'
 #
 # Use the following lists to block specific kinds of file downloads.
 #
@@ -165,6 +165,7 @@ regexpboollist = 'name=exceptionheader,path=@DGCONFDIR@/lists/exceptionregexphea
 regexpreplacelist = 'name=addheader,path=@DGCONFDIR@/lists/addheaderregexplist'
 
 #Virus checking exceptions - matched urls will not be virus checked
+#Note that you also need to amend site.story in order for this to work.
 #mimelist = 'name=exceptionvirus,path=@DGCONFDIR@/lists/contentscanners/exceptionvirusmimetypelist'
 #fileextlist = 'name=exceptionvirus,path=@DGCONFDIR@/lists/contentscanners/exceptionvirusextensionlist'
 #sitelist = 'name=exceptionvirus,path=@DGCONFDIR@/lists/contentscanners/exceptionvirussitelist'

configs/e2guardianf1.conf.in

@@ -5,7 +5,7 @@ AC_DEFINE([__SSLMITM],[""],[Define to enable SSL MITM])
 AC_DEFINE([FD_SETSIZE_OVERIDE],[""],[Define to allow DANS_MAXFD to exceed FD_SETSIZE])
 
 AC_PREREQ(2.57)
-AC_INIT(e2guardian, 5.3.4)
+AC_INIT(e2guardian, 5.3.5)
 AM_INIT_AUTOMAKE
 AC_CONFIG_HEADERS([dgconfig.h])
 AC_CONFIG_MACRO_DIR([m4])
@@ -686,6 +686,7 @@ data/scripts/logrotation
 data/scripts/solaris-init
 data/scripts/systemv-init
 data/scripts/e2guardian.service
+data/scripts/e2guardianrasp.service
 doc/Makefile
 configs/e2guardian.conf
 configs/e2guardianf1.conf

configure.ac

@@ -1,4 +1,4 @@
-This is the v5.3.1 stable version
+This is the v5.3.5 stable version
 
 Note that large sections of the code has been re-written and there are
 significant changes to the configuration files in v5.

notes/NEWIN_v5

@@ -206,8 +206,8 @@ String ConnectionHandler::hashedCookie(String *url, const char *magic, std::stri
     String timecode(bypasstimestamp);
     String data(magic);
     data += clientip->c_str();
-    if(ldl->fg[filtergroup]->bypass_v2)
-            data += clientuser;
+//    if(ldl->fg[filtergroup]->bypass_v2)
+    data += clientuser;
     data += timecode;
 #ifdef DGDEBUG
     std::cerr << thread_id << " -generate Bypass hashedCookie data " << clientip->c_str() << " " << *url << " " << clientuser << " " << timecode << std::endl;
@@ -882,7 +882,7 @@ int ConnectionHandler::handleConnection(Socket &peerconn, String &ip, bool ismit
 
             // is this user banned?
             //isbanneduser = false;
-            if (o.use_xforwardedfor) {
+            if (o.use_xforwardedfor && !ismitm) {  // don't do this for mitm
                 bool use_xforwardedfor;
                 if (o.xforwardedfor_filter_ip.size() > 0) {
                     use_xforwardedfor = false;
@@ -899,6 +899,8 @@ int ConnectionHandler::handleConnection(Socket &peerconn, String &ip, bool ismit
                     std::string xforwardip(header.getXForwardedForIP());
                     if (xforwardip.length() > 6) {
                         clientip = xforwardip;
+                        ip = clientip;
+                        header.setClientIP(ip);
                     }
 #ifdef DGDEBUG
                     std::cerr << thread_id << " -using x-forwardedfor:" << clientip << std::endl;
@@ -1162,7 +1164,9 @@ int ConnectionHandler::handleConnection(Socket &peerconn, String &ip, bool ismit
                 outhead += ud;
                 outhead += "\r\n";
                 outhead += "Location: ";
-                outhead += header.getUrl(true);
+                //outhead += header.getUrl(true);
+                        outhead += checkme.logurl.before("GBYPASS=");
+                        outhead.chop();
                 outhead += "\r\n";
                 outhead += "\r\n";
                 peerconn.writeString(outhead.c_str());
@@ -3084,10 +3088,10 @@ bool ConnectionHandler::checkByPass(NaughtyFilter &checkme, std::shared_ptr<LOpt
         std::cerr << thread_id << " -About to check for bypass..." << std::endl;
 #endif
         if (ldl->fg[filtergroup]->bypass_mode != 0)
-            checkme.bypasstimestamp = isBypassURL(checkme.url, ldl->fg[filtergroup]->magic.c_str(),
+            checkme.bypasstimestamp = isBypassURL(checkme.logurl, ldl->fg[filtergroup]->magic.c_str(),
                                                          clientip.c_str(), NULL, clientuser);
         if ((checkme.bypasstimestamp == 0) && (ldl->fg[filtergroup]->infection_bypass_mode != 0))
-            checkme.bypasstimestamp = isBypassURL(checkme.url, ldl->fg[filtergroup]->imagic.c_str(),
+            checkme.bypasstimestamp = isBypassURL(checkme.logurl, ldl->fg[filtergroup]->imagic.c_str(),
                                                          clientip.c_str(), &checkme.isvirusbypass,
                                                          clientuser);
         if (checkme.bypasstimestamp > 0) {
@@ -3107,7 +3111,11 @@ bool ConnectionHandler::checkByPass(NaughtyFilter &checkme, std::shared_ptr<LOpt
                 checkme.log_message_no = 606;
             }
         } else if (ldl->fg[filtergroup]->bypass_mode != 0) {
-            if (header.isBypassCookie(checkme.urldomain, ldl->fg[filtergroup]->cookie_magic.c_str(),
+            String ud(checkme.urldomain);
+            if (ud.startsWith("www.")) {
+                ud = ud.after("www.");
+            }
+            if (header.isBypassCookie(ud, ldl->fg[filtergroup]->cookie_magic.c_str(),
                                       clientip.c_str(), clientuser.c_str())) {
 #ifdef DGDEBUG
                 std::cerr << thread_id << " -Bypass cookie match" << std::endl;

src/ConnectionHandler.cpp

@@ -156,7 +156,7 @@ bool FOptionContainer::readConfFile(const char *filename) {
     }
     while (!conffiles.eof()) {
         getline(conffiles, linebuffer);
-        if (!conffiles.eof() && linebuffer.length() != 0) {
+        if (!conffiles.fail() && linebuffer.length() != 0) {
             if (linebuffer[0] != '#') { // i.e. not commented out
                 temp = (char *) linebuffer.c_str();
                 if (temp.contains("#")) {

src/FOptionContainer.cpp

@@ -119,6 +119,8 @@ bool StoryBoard::readFile(const char *filename, ListMeta &LM, bool is_top) {
         params = line.before(")").after("(");
         params.removeWhiteSpace();
         action = line.after(")");
+        if (action.contains("#"))
+            action = action.before("#");  // remove trailing comments
         action.removeWhiteSpace();
         if (command == "function") {
             if (in_function) {    // already in another function definition & so assume end of previous function