[SC-67303] Add modification_time field to dbfs FileInfo Object (#360)

Add a new `modification_time` Long field to the `FileInfo` object, received from the DBFS API and exposed in the result of commands such as `dbfs ls -l <path>` or `databricks fs ls -l <path>`.

Author: stormwindy

databricks_cli/dbfs/api.py

@@ -45,29 +45,37 @@ class ParseException(Exception):
 
 
 class FileInfo(object):
-    def __init__(self, dbfs_path, is_dir, file_size):
+    def __init__(self, dbfs_path, is_dir, file_size, modification_time):
         self.dbfs_path = dbfs_path
         self.is_dir = is_dir
         self.file_size = file_size
+        self.modification_time = modification_time
 
     def to_row(self, is_long_form, is_absolute):
         path = self.dbfs_path.absolute_path if is_absolute else self.dbfs_path.basename
         stylized_path = click.style(path, 'cyan') if self.is_dir else path
         if is_long_form:
             filetype = 'dir' if self.is_dir else 'file'
-            return [filetype, self.file_size, stylized_path]
+            row = [filetype, self.file_size, stylized_path]
+            # Add modification time if it is available.
+            if self.modification_time is not None:
+                row.append(self.modification_time)
+            return row
         return [stylized_path]
 
     @classmethod
     def from_json(cls, json):
         dbfs_path = DbfsPath.from_api_path(json['path'])
-        return cls(dbfs_path, json['is_dir'], json['file_size'])
+        # If JSON doesn't include modification_time data, replace it with None.
+        modification_time = json['modification_time'] if 'modification_time' in json else None
+        return cls(dbfs_path, json['is_dir'], json['file_size'], modification_time)
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):
             return self.dbfs_path == other.dbfs_path and \
                 self.is_dir == other.is_dir and \
-                self.file_size == other.file_size
+                self.file_size == other.file_size and \
+                self.modification_time == other.modification_time
         return False
 
 

databricks_cli/dbfs/cli.py

@@ -36,7 +36,8 @@
 @click.option('--absolute', is_flag=True, default=False,
               help='Displays absolute paths.')
 @click.option('-l', is_flag=True, default=False,
-              help='Displays full information including size and file type.')
+              help="""Displays full information including size, file type 
+                      and modification time since Epoch in milliseconds.""")
 @click.argument('dbfs_path', nargs=-1, type=DbfsPathClickType())
 @debug_option
 @profile_option

tests/dbfs/test_api.py

@@ -25,6 +25,7 @@
 from base64 import b64encode
 
 import os
+
 import requests
 import mock
 import pytest
@@ -34,12 +35,14 @@
 from databricks_cli.dbfs.exceptions import LocalFileExistsException
 
 TEST_DBFS_PATH = DbfsPath('dbfs:/test')
+DUMMY_TIME = 1613158406000
 TEST_FILE_JSON = {
     'path': '/test',
     'is_dir': False,
-    'file_size': 1
+    'file_size': 1,
+    'modification_time': DUMMY_TIME
 }
-TEST_FILE_INFO = api.FileInfo(TEST_DBFS_PATH, False, 1)
+TEST_FILE_INFO = api.FileInfo(TEST_DBFS_PATH, False, 1, DUMMY_TIME)
 
 
 def get_resource_does_not_exist_exception():
@@ -57,15 +60,15 @@ def get_partial_delete_exception(message="[...] operation has deleted 10 files [
 
 class TestFileInfo(object):
     def test_to_row_not_long_form_not_absolute(self):
-        file_info = api.FileInfo(TEST_DBFS_PATH, False, 1)
+        file_info = api.FileInfo(TEST_DBFS_PATH, False, 1, DUMMY_TIME)
         row = file_info.to_row(is_long_form=False, is_absolute=False)
         assert len(row) == 1
         assert TEST_DBFS_PATH.basename == row[0]
 
     def test_to_row_long_form_not_absolute(self):
-        file_info = api.FileInfo(TEST_DBFS_PATH, False, 1)
+        file_info = api.FileInfo(TEST_DBFS_PATH, False, 1, DUMMY_TIME)
         row = file_info.to_row(is_long_form=True, is_absolute=False)
-        assert len(row) == 3
+        assert len(row) == 4
         assert row[0] == 'file'
         assert row[1] == 1
         assert TEST_DBFS_PATH.basename == row[2]
@@ -163,7 +166,8 @@ def test_cat(self, dbfs_api):
         dbfs_api.client.get_status.return_value = {
             'path': '/test',
             'is_dir': False,
-            'file_size': 1
+            'file_size': 1,
+            'modification_time': DUMMY_TIME
         }
         dbfs_api.client.read.return_value = {
             'bytes_read': 1,