Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions lanforge_client/api_call_logger.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#!/usr/bin/env python3
"""
Shared, lightweight API call logger.

This exists so any code path that talks to a LANforge GUI -- the legacy
py-json/LANforge/LFRequest.py request layer (used via LFCliBase/Realm), or
lanforge_api.py's own request layer -- can record the same kind of entry
(url, payload, response code, session id) to the same kind of rotating log
file, without each path reimplementing rotation/formatting itself.
"""
import os
import json
import logging
import logging.handlers
import datetime

_api_loggers = {}


def get_api_logger(filename, max_bytes=10 * 1024 * 1024, backup_count=10):
"""
Return a cached, rotating file logger for `filename`. Callers that pass the same
filename (even from different LFCliBase/Realm instances, or a different request layer
entirely) share one logger/handler, so log lines are never duplicated and the file
rotates once instead of racing multiple independent handlers against each other.
:param filename: path to the api log file
:param max_bytes: rotate once the active file reaches this size
:param backup_count: number of rotated backups to keep
:return: a logging.Logger configured with a RotatingFileHandler
"""
key = os.path.abspath(filename)
if key in _api_loggers:
return _api_loggers[key]
api_logger = logging.getLogger("lanforge_client.api_call_logger.%s" % key)
api_logger.setLevel(logging.INFO)
api_logger.propagate = False
handler = logging.handlers.RotatingFileHandler(filename, maxBytes=max_bytes, backupCount=backup_count)
handler.setFormatter(logging.Formatter("%(message)s"))
api_logger.addHandler(handler)
_api_loggers[key] = api_logger
return api_logger


def log_api_call(api_logger, method, url, session_id=None, data=None, response_code=None, error=None,
diagnostics=None):
"""
Format and emit one lightweight record of an API call via api_logger. No-op if
api_logger is None, so callers can pass through a possibly-absent logger without
guarding every call site themselves.
:param api_logger: a logger from get_api_logger(), or None to skip silently
:param method: "GET" | "POST" | "PUT" | "DELETE"
:param url: requested url
:param session_id: the lanforge_api session id ("cookie") tying this call to the
LANforge GUI's own request logs, if available
:param data: payload sent (POST/PUT only)
:param response_code: HTTP status code returned by the call, if any
:param error: exception raised by the call, if any -- marks the entry as ERROR
:param diagnostics: one-line summary of a caught HTTPError/URLError (reason,
X-Error-* headers, etc.)
"""
if api_logger is None:
return
if error is not None:
status = "ERROR"
elif response_code is not None:
status = "OK" if 200 <= response_code < 300 else "ERROR"

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Treat expected LANforge statuses as successful

The request layer explicitly treats several non-2xx responses as acceptable (BaseLFJsonRequest.OK_STATUSES includes redirects and 404 for normal polling/missing-resource cases), but this logger marks anything outside 2xx as ERROR. In workflows that expect temporary 404s while waiting for resources, the saved API log/report will report normal calls as failures and obscure real errors; align this status calculation with the request layer's accepted status set.

Useful? React with 👍 / 👎.

else:
status = "UNKNOWN"
lines = ["%s session=%s %s %s [%s]" %
(datetime.datetime.now().isoformat(), session_id or '-', method, url, status)]
if data is not None:
lines.append(" payload: %s" % json.dumps(data, default=str))
if error is not None:
lines.append(" error: %s" % error)
if response_code is not None:
lines.append(" response_code: %s" % response_code)
if diagnostics is not None:
lines.append(" diagnostics: %s" % diagnostics)
try:
api_logger.info("\n".join(lines))
except Exception:
logging.getLogger(__name__).debug("log_api_call: unable to emit log line", exc_info=True)
128 changes: 98 additions & 30 deletions lanforge_client/lanforge_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ class which appends subclasses to it.

# - - - - deployed import references - - - - -
from .strutil import nott, iss
from .api_call_logger import get_api_logger, log_api_call

SESSION_HEADER = 'X-LFJson-Session'
# LOGGER = Logger('json_api')
Expand Down Expand Up @@ -173,6 +174,10 @@ def print_diagnostics(url_: str = None,
error_list_.append(xerr)
LOGGER.error(" = = = = = = = = = = = = = = = =")

summary = "%s <%s> HTTP %s: %s" % (method, err_full_url, err_code, err_reason)
if xerrors and err_code != 404:
summary += " | " + "; ".join(xerrors)

if error_.__class__ is urllib.error.HTTPError:
LOGGER.debug("----- HTTPError: ------------------------------------ print_diagnostics:")
LOGGER.debug("%s <%s> HTTP %s: %s" % (method, err_full_url, err_code, err_reason))
Expand Down Expand Up @@ -203,14 +208,15 @@ def print_diagnostics(url_: str = None,
LOGGER.warning("------------------------------------------------------------------------")
if die_on_error_:
exit(1)
return
return summary

if error_.__class__ is urllib.error.URLError:
LOGGER.error("----- URLError: ---------------------------------------------")
LOGGER.error("%s <%s> HTTP %s: %s" % (method, err_full_url, err_code, err_reason))
LOGGER.error("------------------------------------------------------------------------")
if die_on_error_:
exit(1)
return summary


class BaseLFJsonRequest:
Expand Down Expand Up @@ -242,6 +248,10 @@ def __init__(self,
self.session_instance = None
self.stream_errors: bool = True
self.stream_warnings: bool = False
# set by get()/json_post() so _log_api_call() can report the outcome of the last
# attempt even when it ended in a caught HTTPError/URLError
self.last_response_code = None
self.last_diagnostics = None

if not session_obj:
logging.getLogger(__name__).warning("BaseLFJsonRequest: no session instance")
Expand Down Expand Up @@ -320,6 +330,18 @@ def add_warning(self, message: str = None):
def get_errors(self) -> list:
return self.error_list

def _log_api_call(self, method, url, data=None, response_code=None, diagnostics=None):
"""
Record one json_get/json_post/json_put/json_delete call via the shared
lanforge_client.api_call_logger, tagged with this session's id. No-op unless
self.session_instance.save_api is True.
"""
if not getattr(self.session_instance, 'save_api', False):
return
log_api_call(getattr(self.session_instance, '_api_logger', None), method, url,
session_id=self.session_instance.get_session_id(),
data=data, response_code=response_code, diagnostics=diagnostics)

def get_warnings(self) -> list:
return self.warnings

Expand Down Expand Up @@ -635,20 +657,25 @@ def json_post(self,
self.logger.debug("----------------- BAD STATUS --------------------------------")
if die_on_error:
sys.exit(1)
self.last_response_code = response.status
self._log_api_call(method_, url, data=post_data, response_code=self.last_response_code)
return responses[0]

except urllib.error.HTTPError as herror:
# these error codes illustrate an error on the client that requires debugging
# and retrying them is never going to succeed
if herror.code in (400, 410, 411, 412, 413, 414, 415, 416, 417, 428, 429, 431, 451):
die_on_error = True
print_diagnostics(url_=url,
request_=myrequest,
responses_=responses,
error_=herror,
debug_=debug,
die_on_error_=die_on_error)
self.last_response_code = herror.code
self.last_diagnostics = print_diagnostics(url_=url,
request_=myrequest,
responses_=responses,
error_=herror,
debug_=debug,
die_on_error_=die_on_error)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Log fatal API failures before exiting

When lanforge_api hits a fatal error path, such as a 400-class response that sets die_on_error = True or a caller that requested die_on_error, this call passes that flag into print_diagnostics; that helper calls exit(1) before returning in this mode, so the _log_api_call immediately below is never reached. With save_api enabled, the failures that abort the script are absent from the API log; log before invoking an exiting helper or call it without exiting and exit after logging.

Useful? React with 👍 / 👎.

if die_on_error:
self._log_api_call(method_, url, data=post_data, response_code=self.last_response_code,
diagnostics=self.last_diagnostics)
sys.exit(1)

except urllib.error.URLError as uerror:
Expand All @@ -660,17 +687,21 @@ def json_post(self,
break
else:
logging.error("Connection refused: "+url)
print_diagnostics(url_=url,
request_=myrequest,
responses_=responses,
error_=uerror,
debug_=debug,
die_on_error_=die_on_error)
self.last_diagnostics = print_diagnostics(url_=url,
request_=myrequest,
responses_=responses,
error_=uerror,
debug_=debug,
die_on_error_=die_on_error)
if die_on_error:
self._log_api_call(method_, url, data=post_data, response_code=self.last_response_code,
diagnostics=self.last_diagnostics)
sys.exit(1)
# ~while
LOGGER.error("json_post: request will try again in 2 sec")
time.sleep(2)
self._log_api_call(method_, url, data=post_data, response_code=self.last_response_code,
diagnostics=self.last_diagnostics)
if die_on_error:
sys.exit(1)
return None
Expand Down Expand Up @@ -800,26 +831,28 @@ def get(self,
myresponses: list = [] # list[HTTPResponse]
try:
myresponses.append(request.urlopen(myrequest))
self.last_response_code = getattr(myresponses[0], 'status', None)
return myresponses[0]

except urllib.error.HTTPError as herror:
print_diagnostics(url_=requested_url,
request_=myrequest,
responses_=myresponses,
error_=herror,
error_list_=self.error_list,
debug_=debug,
die_on_error_=die_on_error)
self.last_response_code = herror.code
self.last_diagnostics = print_diagnostics(url_=requested_url,
request_=myrequest,
responses_=myresponses,
error_=herror,
error_list_=self.error_list,
debug_=debug,
die_on_error_=die_on_error)
if die_on_error:
sys.exit(1)
except urllib.error.URLError as uerror:
print_diagnostics(url_=requested_url,
request_=myrequest,
responses_=myresponses,
error_=uerror,
error_list_=self.error_list,
debug_=debug,
die_on_error_=die_on_error)
self.last_diagnostics = print_diagnostics(url_=requested_url,
request_=myrequest,
responses_=myresponses,
error_=uerror,
error_list_=self.error_list,
debug_=debug,
die_on_error_=die_on_error)
if die_on_error:
sys.exit(1)
if die_on_error:
Expand Down Expand Up @@ -863,6 +896,8 @@ def get_as_json(self,
if responses[0] is None:
if debug:
self.logger.debug(msg="No response from " + url)
self._log_api_call(method_, url, response_code=self.last_response_code,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Reset per-request API log state

Because last_response_code and last_diagnostics are instance fields and are not cleared at the start of each request, a reused LFJsonQuery can carry a previous response code into a later call that gets no response, such as a URLError. This log entry then writes the stale code/status for the new failure, so the API log can show a failed call as response_code: 200/OK; reset these fields before each request or set the code to None on no-response paths.

Useful? React with 👍 / 👎.

diagnostics=self.last_diagnostics)
return None

json_data = json.loads(responses[0].read().decode('utf-8'))
Expand All @@ -872,6 +907,8 @@ def get_as_json(self,
if "warnings" in responses[0]:
errors_warnings.extend(json_data["warnings"])

self._log_api_call(method_, url, response_code=self.last_response_code,
diagnostics=self.last_diagnostics)
return json_data

def json_get(self,
Expand Down Expand Up @@ -1179,10 +1216,26 @@ def __init__(self, lfclient_url: str = 'http://localhost:8080',
retry_sec: float = Default_Retry_Sec,
stream_errors: bool = True,
stream_warnings: bool = False,
exit_on_error: bool = False):
exit_on_error: bool = False,
save_api: bool = False,
api_log_file_name: str = None,
api_log_max_bytes: int = 10 * 1024 * 1024,
api_log_backup_count: int = 10):
self.debug_on = debug
# self.logger = Logg(name='json_api_session')
self.logger = logging.getLogger(__name__)
# when True, json_get/json_post/json_put/json_delete append a lightweight record of
# each call (url, payload, response_code/error) to api_log_filename, tagged with this
# session's id -- see lanforge_client/api_call_logger.py. The file rotates instead of
# growing unbounded, so it's safe to leave enabled indefinitely.
self.save_api = save_api
self.api_log_filename = api_log_file_name or os.path.join(os.path.expanduser('~'), 'lf_api_calls.log')
self._api_logger = None
if self.save_api:
try:
self._api_logger = get_api_logger(self.api_log_filename, api_log_max_bytes, api_log_backup_count)
except Exception as x:
self.logger.debug("BaseSession: unable to set up api logger for %s: %s" % (self.api_log_filename, x))

if debug:
self.logger.level = logging.DEBUG
Expand Down Expand Up @@ -25531,7 +25584,11 @@ def __init__(self, lfclient_url: str = 'http://localhost:8080',
stream_errors: bool = True,
stream_warnings: bool = False,
require_session: bool = False,
exit_on_error: bool = False):
exit_on_error: bool = False,
save_api: bool = False,
api_log_file_name: str = None,
api_log_max_bytes: int = 10 * 1024 * 1024,
api_log_backup_count: int = 10):
"""
:param debug: turn on diagnostic information
:param proxy_map: a dict with addresses of proxies to route requests through.
Expand All @@ -25544,14 +25601,25 @@ def __init__(self, lfclient_url: str = 'http://localhost:8080',
:param require_session: exit(1) if unable to establish a session_id
:param exit_on_error: on requests failing HTTP requests on besides error 404,
exit(1). This does not include failing to establish a session_id
:param save_api: when True, append a lightweight record of every json_get/json_post/
json_put/json_delete call (url, payload, response_code/error) to api_log_file_name,
tagged with this session's id
:param api_log_file_name: path to the api log file used when save_api is True
(default: ~/lf_api_calls.log)
:param api_log_max_bytes: rotate the api log once it reaches this size
:param api_log_backup_count: number of rotated api log backups to keep
"""
super().__init__(lfclient_url=lfclient_url,
debug=debug,
proxy_map=proxy_map,
connection_timeout_sec=connection_timeout_sec,
stream_errors=stream_errors,
stream_warnings=stream_warnings,
exit_on_error=exit_on_error)
exit_on_error=exit_on_error,
save_api=save_api,
api_log_file_name=api_log_file_name,
api_log_max_bytes=api_log_max_bytes,
api_log_backup_count=api_log_backup_count)
self.command_instance = LFJsonCommand(session_obj=self, debug=debug, exit_on_error=exit_on_error)
self.session_connection_check = \
self.command_instance.start_session(debug=debug,
Expand Down
13 changes: 12 additions & 1 deletion py-json/LANforge/LFRequest.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@

debug_printer = PrettyPrinter(indent=2)

# must match lanforge_client.lanforge_api.SESSION_HEADER -- this is the header the LANforge
# GUI uses to correlate a REST request with a session id in its own logs
SESSION_HEADER = 'X-LFJson-Session'


class LFRequest:
Default_Base_URL = "http://localhost:8080"
Expand All @@ -37,12 +41,19 @@ def __init__(self, url=None,
uri=None,
proxies_=None,
debug_=False,
die_on_error_=False):
die_on_error_=False,
session_id_=None):
self.debug = debug_
self.die_on_error = die_on_error_
self.error_list = []
self.last_response_code = None
self.last_diagnostics = None
self.session_id = session_id_
# instance-level copy: default_headers is a class attribute (shared dict), so mutating
# it in place here would leak this instance's session id into every other LFRequest
self.default_headers = dict(LFRequest.default_headers)
if session_id_:
self.default_headers[SESSION_HEADER] = session_id_

# please see this discussion on ProxyHandlers:
# https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler
Expand Down
Loading
Loading