Add reconnect loop with exponential backoff to async_main

Author: Pyker

irc_bot.py

@@ -4,6 +4,7 @@
 import logging
 import signal
 import socket
+import time
 import traceback
 
 from command_router import CommandRouter
@@ -185,46 +186,66 @@ async def async_main():
     config_obj.check_options()
 
     bot = IrcBot(config_obj)
-    log = False
-
-    try:
-        await bot.start()
-    except Exception as error:
-        if getattr(bot, "_logger", None) is not None:
-            bot._logger.exception("Fatal error during startup or runtime.")
-            log = True
-        else:
-            print("Logger was not initialized, cannot write to log file.")
-
-        print(f"Fatal error: {type(error).__name__}: {error}")
-        traceb = traceback.format_exc()
-        print(traceb)
-
-        with open("exception.txt", "w") as excFile:
-            excFile.write(f"Fatal error at {datetime.datetime.today()}\n")
-            excFile.write(traceb + "\n")
-            excFile.write("-----------------------------------------------------\n")
-
-            if getattr(bot, "command_router", None) is not None:
-                while not bot.command_router.recent_messages.empty():
-                    msg = bot.command_router.recent_messages.get_nowait()
-                    excFile.write(msg)
-                    excFile.write("\n")
-                bot.command_router.task_pool.cancel_all()
-                if log:
-                    bot._logger.debug("All tasks were signaled to shut down.")
-
-            excFile.write("-----------------------------------------------------\n")
-            excFile.write("Connection Exception: \n")
-
-            if getattr(bot, "conn", None) is not None:
-                excFile.write(str(bot.conn.error) + " \n")
+
+    initial_backoff = 5
+    max_backoff = 300
+    stable_threshold = 60
+    backoff = initial_backoff
+
+    while True:
+        start_time = time.monotonic()
+        try:
+            await bot.start()
+            break
+
+        except (ConnectionDown, OSError, TimeoutError) as error:
+            elapsed = time.monotonic() - start_time
+            if elapsed > stable_threshold:
+                backoff = initial_backoff
+            bot._logger.warning("Connection lost: %s. Reconnecting in %ds...", error, backoff)
+            await asyncio.sleep(backoff)
+            backoff = min(backoff * 2, max_backoff)
+
+        except asyncio.CancelledError:
+            break
+
+        except Exception as error:
+            if getattr(bot, "_logger", None) is not None:
+                bot._logger.exception("Fatal error during startup or runtime.")
             else:
-                excFile.write("Connection not initialized\n")
+                print("Logger was not initialized, cannot write to log file.")
+
+            print(f"Fatal error: {type(error).__name__}: {error}")
+            traceb = traceback.format_exc()
+            print(traceb)
+
+            with open("exception.txt", "w") as excFile:
+                excFile.write(f"Fatal error at {datetime.datetime.today()}\n")
+                excFile.write(traceb + "\n")
+                excFile.write("-----------------------------------------------------\n")
+
+                if getattr(bot, "command_router", None) is not None:
+                    while not bot.command_router.recent_messages.empty():
+                        msg = bot.command_router.recent_messages.get_nowait()
+                        excFile.write(msg)
+                        excFile.write("\n")
+                    bot.command_router.task_pool.cancel_all()
+                    if getattr(bot, "_logger", None) is not None:
+                        bot._logger.debug("All tasks were signaled to shut down.")
+
+                excFile.write("-----------------------------------------------------\n")
+                excFile.write("Connection Exception: \n")
+
+                if getattr(bot, "conn", None) is not None:
+                    excFile.write(str(bot.conn.error) + " \n")
+                else:
+                    excFile.write("Connection not initialized\n")
+
+                excFile.write("-----------------------------------------------------\n")
 
-            excFile.write("-----------------------------------------------------\n")
+            break
 
-    if log:
+    if getattr(bot, "_logger", None) is not None:
         bot._logger.info("End of Session\n\n\n\n")
     logging.shutdown()
 

tests/test_irc_bot.py

@@ -201,3 +201,170 @@ async def test_nickserv_auth_reset_on_start(self):
             await bot.start()
 
         assert bot.nickserv_auth is False
+
+
+class TestAsyncMainReconnect:
+    def _mock_config(self):
+        """Mock Configuration so async_main doesn't need config.yml."""
+        config_obj = MagicMock()
+        config_obj.config = {
+            "connection": {
+                "server": "irc.example.com",
+                "port": 6667,
+                "nickname": "TestBot",
+                "password": "",
+                "ident": "testbot",
+                "realname": "Test",
+            },
+            "administration": {
+                "operators": [],
+                "channels": [],
+                "command_prefix": "!",
+                "logging_level": "INFO",
+            },
+            "networking": {"force_ipv6": False, "bind_address": ""},
+        }
+        return config_obj
+
+    async def test_retries_on_connection_down(self):
+        from irc_bot import async_main
+
+        config_obj = self._mock_config()
+        call_count = 0
+
+        async def fake_start(self):
+            nonlocal call_count
+            call_count += 1
+            if call_count < 3:
+                raise ConnectionDown("irc.example.com", "now")
+            # Third call: simulate graceful shutdown
+            self.shutdown = True
+
+        with (
+            patch("irc_bot.Configuration") as MockConfig,
+            patch.object(IrcBot, "start", fake_start),
+            patch("irc_bot.write_starting_date"),
+            patch("irc_bot.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
+            patch("irc_bot.time") as mock_time,
+        ):
+            MockConfig.return_value = config_obj
+            mock_time.monotonic.return_value = 0  # Elapsed < stable_threshold
+            await async_main()
+
+        assert call_count == 3
+        assert mock_sleep.await_count == 2  # Two retries before success
+
+    async def test_backoff_doubles(self):
+        from irc_bot import async_main
+
+        config_obj = self._mock_config()
+        call_count = 0
+
+        async def fake_start(self):
+            nonlocal call_count
+            call_count += 1
+            if call_count < 4:
+                raise ConnectionDown("irc.example.com", "now")
+            self.shutdown = True
+
+        with (
+            patch("irc_bot.Configuration") as MockConfig,
+            patch.object(IrcBot, "start", fake_start),
+            patch("irc_bot.write_starting_date"),
+            patch("irc_bot.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
+            patch("irc_bot.time") as mock_time,
+        ):
+            MockConfig.return_value = config_obj
+            mock_time.monotonic.return_value = 0
+            await async_main()
+
+        # Backoff: 5, 10, 20
+        sleep_values = [c.args[0] for c in mock_sleep.call_args_list]
+        assert sleep_values == [5, 10, 20]
+
+    async def test_backoff_caps_at_300(self):
+        from irc_bot import async_main
+
+        config_obj = self._mock_config()
+        call_count = 0
+
+        async def fake_start(self):
+            nonlocal call_count
+            call_count += 1
+            if call_count < 10:
+                raise ConnectionDown("irc.example.com", "now")
+            self.shutdown = True
+
+        with (
+            patch("irc_bot.Configuration") as MockConfig,
+            patch.object(IrcBot, "start", fake_start),
+            patch("irc_bot.write_starting_date"),
+            patch("irc_bot.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
+            patch("irc_bot.time") as mock_time,
+        ):
+            MockConfig.return_value = config_obj
+            mock_time.monotonic.return_value = 0
+            await async_main()
+
+        sleep_values = [c.args[0] for c in mock_sleep.call_args_list]
+        # 5, 10, 20, 40, 80, 160, 300, 300, 300
+        assert sleep_values[-1] == 300
+        assert all(v <= 300 for v in sleep_values)
+
+    async def test_backoff_resets_after_stable_connection(self):
+        from irc_bot import async_main
+
+        config_obj = self._mock_config()
+        call_count = 0
+        # 9 monotonic() calls: 5 for start_time + 4 for elapsed.
+        # Iteration 3's elapsed = 100 - 0 = 100 > stable_threshold, triggers reset.
+        times = iter([0, 0, 0, 0, 0, 100, 0, 0, 0])
+
+        async def fake_start(self):
+            nonlocal call_count
+            call_count += 1
+            if call_count < 5:
+                raise ConnectionDown("irc.example.com", "now")
+            self.shutdown = True
+
+        with (
+            patch("irc_bot.Configuration") as MockConfig,
+            patch.object(IrcBot, "start", fake_start),
+            patch("irc_bot.write_starting_date"),
+            patch("irc_bot.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
+            patch("irc_bot.time") as mock_time,
+        ):
+            MockConfig.return_value = config_obj
+            mock_time.monotonic.side_effect = times
+            await async_main()
+
+        sleep_values = [c.args[0] for c in mock_sleep.call_args_list]
+        # First two failures: 5, 10. Third was stable (100s elapsed), resets to 5. Fourth: 10.
+        assert sleep_values == [5, 10, 5, 10]
+
+    async def test_retries_on_oserror(self):
+        from irc_bot import async_main
+
+        config_obj = self._mock_config()
+        call_count = 0
+
+        async def fake_start(self):
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                raise ConnectionRefusedError("Connection refused")
+            self.shutdown = True
+
+        with (
+            patch("irc_bot.Configuration") as MockConfig,
+            patch.object(IrcBot, "start", fake_start),
+            patch("irc_bot.write_starting_date"),
+            patch("irc_bot.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
+            patch("irc_bot.time") as mock_time,
+        ):
+            MockConfig.return_value = config_obj
+            mock_time.monotonic.return_value = 0
+            await async_main()
+
+        assert call_count == 2
+        assert mock_sleep.await_count == 1