Major Python 3 migration and bug fixes

This commit consolidates extensive Python 3 compatibility improvements,
bug fixes, and feature enhancements including:

- Python 3 migration: Replace raw_input with input, fix execfile usage,
  handle string/bytes encoding throughout codebase
- Encoding fixes: UTF-8 handling, proper encoding for email headers,
  subscription forms, and message decoration
- Archiver improvements: Fix on-the-fly archiving, handle non-ASCII
  characters, improve archive format handling
- cPanel integration: Sync fixes for pickle protocol detection, encoding
  issues, and various CPANEL ticket fixes
- Configuration: Improve mm_cfg handler, fix circular import issues
- Bug fixes: Fix TypeError with string encoding/hashing, KeyError in
  HyperDatabase, NNTP bridge issues, and various other fixes
- Cron script fixes: Update senddigests, checkdbs, and other cron jobs
  for Python 3 compatibility

This represents a comprehensive update bringing the codebase to full
Python 3 compatibility while maintaining backward compatibility where
possible.

Author: jared mauch

Mailman/Archiver/Archiver.py

@@ -29,7 +29,7 @@
 import errno
 import traceback
 import re
-from io import StringIO
+import tempfile
 
 from Mailman import mm_cfg
 from Mailman import Mailbox
@@ -88,17 +88,21 @@ def InitVars(self):
         # symbolic links.
         omask = os.umask(0)
         try:
-            # Create mbox directory with proper permissions
-            mbox_dir = self.archive_dir() + '.mbox'
-            os.makedirs(mbox_dir, mode=0o02775, exist_ok=True)
-            
-            # Create archive directory with proper permissions
-            archive_dir = self.archive_dir()
-            os.makedirs(archive_dir, mode=0o02775, exist_ok=True)
-            
+            try:
+                os.mkdir(self.archive_dir()+'.mbox', 0o02775)
+            except OSError as e:
+                if e.errno != errno.EEXIST: raise
+                # We also create an empty pipermail archive directory into
+                # which we'll drop an empty index.html file into.  This is so
+                # that lists that have not yet received a posting have
+                # /something/ as their index.html, and don't just get a 404.
+            try:
+                os.mkdir(self.archive_dir(), 0o02775)
+            except OSError as e:
+                if e.errno != errno.EEXIST: raise
             # See if there's an index.html file there already and if not,
             # write in the empty archive notice.
-            indexfile = os.path.join(archive_dir, 'index.html')
+            indexfile = os.path.join(self.archive_dir(), 'index.html')
             fp = None
             try:
                 fp = open(indexfile)
@@ -132,7 +136,8 @@ def GetBaseArchiveURL(self):
         if self.archive_private:
             return url
         else:
-            hostname = re.match(r'[^:]*://([^/]*)/.*', url, re.IGNORECASE).group(1)
+            hostname = re.match('[^:]*://([^/]*)/.*', url).group(1)\
+                       or mm_cfg.DEFAULT_URL_HOST
             url = mm_cfg.PUBLIC_ARCHIVE_URL % {
                 'listname': self.internal_name(),
                 'hostname': hostname
@@ -145,7 +150,7 @@ def __archive_file(self, afn):
         """Open (creating, if necessary) the named archive file."""
         omask = os.umask(0o002)
         try:
-            return Mailbox.Mailbox(open(afn, 'a+'))
+            return Mailbox.Mailbox(open(afn, 'a+b'))
         finally:
             os.umask(omask)
 
@@ -157,9 +162,11 @@ def __archive_to_mbox(self, post):
         """Retain a text copy of the message in an mbox file."""
         try:
             afn = self.ArchiveFileName()
+            syslog('debug', 'Archiver: Writing to mbox file: %s', afn)
             mbox = self.__archive_file(afn)
             mbox.AppendMessage(post)
-            mbox.fp.close()
+            mbox.close()
+            syslog('debug', 'Archiver: Successfully wrote message to mbox file: %s', afn)
         except IOError as msg:
             syslog('error', 'Archive file access failure:\n\t%s %s', afn, msg)
             raise
@@ -169,55 +176,68 @@ def ExternalArchive(self, ar, txt):
                       'hostname': self.host_name,
                       })
         cmd = ar % d
-        try:
-            with os.popen(cmd, 'w') as extarch:
-                extarch.write(txt)
-        except OSError as e:
-            syslog('error', 'Failed to execute external archiver: %s\nError: %s',
-                   cmd, str(e))
-            return
+        extarch = os.popen(cmd, 'w')
+        extarch.write(txt)
         status = extarch.close()
         if status:
-            syslog('error', 'External archiver non-zero exit status: %d\nCommand: %s',
-                   (status & 0xff00) >> 8, cmd)
+            syslog('error', 'external archiver non-zero exit status: %d\n',
+                   (status & 0xff00) >> 8)
 
     #
     # archiving in real time  this is called from list.post(msg)
     #
     def ArchiveMail(self, msg):
         """Store postings in mbox and/or pipermail archive, depending."""
+        from Mailman.Logging.Syslog import syslog
+        syslog('debug', 'Archiver: Starting ArchiveMail for list %s', self.internal_name())
+        
         # Fork so archival errors won't disrupt normal list delivery
         if mm_cfg.ARCHIVE_TO_MBOX == -1:
+            syslog('debug', 'Archiver: ARCHIVE_TO_MBOX is -1, archiving disabled')
             return
+        
+        syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = %s', mm_cfg.ARCHIVE_TO_MBOX)
         #
         # We don't need an extra archiver lock here because we know the list
         # itself must be locked.
         if mm_cfg.ARCHIVE_TO_MBOX in (1, 2):
-            try:
-                mbox = self.__archive_file(self.ArchiveFileName())
-                mbox.AppendMessage(msg)
-                mbox.fp.close()
-            except IOError as msg:
-                syslog('error', 'Archive file access failure:\n\t%s %s', 
-                       self.ArchiveFileName(), msg)
-                raise
+            syslog('debug', 'Archiver: Writing to mbox archive')
+            self.__archive_to_mbox(msg)
             if mm_cfg.ARCHIVE_TO_MBOX == 1:
                 # Archive to mbox only.
+                syslog('debug', 'Archiver: ARCHIVE_TO_MBOX = 1, mbox only, returning')
                 return
-        txt = str(msg)
+
+        txt = msg.as_string()
+        unixfrom = msg.get_unixfrom()
+        # Handle case where unixfrom is None (Python 3 compatibility)
+        if unixfrom and not txt.startswith(unixfrom):
+            txt = unixfrom + '\n' + txt
+
         # should we use the internal or external archiver?
         private_p = self.archive_private
+        syslog('debug', 'Archiver: archive_private = %s', private_p)
+        
         if mm_cfg.PUBLIC_EXTERNAL_ARCHIVER and not private_p:
+            syslog('debug', 'Archiver: Using public external archiver')
             self.ExternalArchive(mm_cfg.PUBLIC_EXTERNAL_ARCHIVER, txt)
         elif mm_cfg.PRIVATE_EXTERNAL_ARCHIVER and private_p:
+            syslog('debug', 'Archiver: Using private external archiver')
             self.ExternalArchive(mm_cfg.PRIVATE_EXTERNAL_ARCHIVER, txt)
         else:
             # use the internal archiver
-            with StringIO(txt) as f:
-                from . import HyperArch
-                h = HyperArch.HyperArchive(self)
-                h.processUnixMailbox(f)
-                h.close()
+            syslog('debug', 'Archiver: Using internal HyperArch archiver')
+            f = tempfile.NamedTemporaryFile()
+            if isinstance(txt, str):
+                txt = txt.encode('utf-8')
+            f.write(txt)
+            f.flush()
+            from . import HyperArch
+            h = HyperArch.HyperArchive(self)
+            h.processUnixMailbox(f)
+            h.close()
+            f.close()
+            syslog('debug', 'Archiver: Completed internal archiving')
 
     #
     # called from MailList.MailList.Save()