Merge pull request #1 from alexnad/storage_engine

Storage engine

Author: alexnad

src/storage/__init__.py

@@ -0,0 +1,311 @@
+import struct
+
+
+class StorageHandlingException(Exception):
+    def __init__(self, message, **kwargs):
+        self.message = message
+        self.kwargs = kwargs
+
+    def __str__(self):
+        return self.message.format(self.kwargs)
+
+
+class FormatStringNotSupportedError(StorageHandlingException):
+    def __init__(self, format_string):
+        self.message = "Format string: '{format_sting}' not supported"
+        super().__init__(self, self.message, format_string)
+
+
+class FormatStringError(StorageHandlingException):
+    def __init__(self, format_string, table_length, map_size):
+        self.message = "Format string '{format_string}' byte size " \
+                       "and table length:{table_length} " +\
+                       "do not match allocation map size {map_size}"
+        super().__init__(self, self.message, format_string,
+                         table_length, map_size)
+
+
+class BlockOutOfRangeError(StorageHandlingException):
+    def __init__(self, block, allocation_map_size):
+        self.message = "Block:{block} is greater than the " + \
+                       "allocation map size:{allocation_map_size}"
+        super().__init__(self, block, allocation_map_size)
+
+
+class GlobalAllocationMapMixin:
+    """
+    This class defines basic functionality
+    for global allocation maps keeping
+    track and managing empty blocks in a file
+    """
+
+    def create_empty_table(self, number_of_elements):
+        return (0 for i in range(number_of_elements))
+
+    def get_free_blocks(self, table, number_of_needed_blocks):
+        """
+        Takes an allocation map
+        Args:
+            table (iterable(int)): representing the information in the
+                                   allocation map should be an integer
+                                   in range [-128;127]
+            number_of_needed_blocks (int): how many free blocks are needed
+        Returns
+            list(int): numbers of free blocks
+        """
+        result = []
+        for index, byte in enumerate(table):
+            if byte == 255:
+                continue
+            free_blocks_in_byte = self.free_bits[byte]
+
+            while free_blocks_in_byte:
+                result.append(free_blocks_in_byte.pop(0))
+
+                if len(result) >= number_of_needed_blocks:
+                    break
+            if len(result) >= number_of_needed_blocks:
+                break
+
+        bits_in_byte = 8
+        offset = bits_in_byte * index
+
+        return ((offset + bit) for bit in result)
+
+    def get_block_location(self, block):
+        if block >= self.allocation_map_size:
+            raise BlockOutOfRangeError(block, self.allocation_map_size)
+
+        byte = block // 8
+        bit = block % 8
+
+        return byte, bit
+
+    def bit_flag(self, bit):
+        flags = [128, 64, 32, 16, 8, 4, 2, 1]
+        return flags[bit]
+
+    def free_block(self, block, table):
+        """
+        Frees block by setting corresponding bit in map to 0
+        Args:
+            block (int): place of block in table
+            table (list(int)): list from ints in range [-128;127]
+                               each int represent a byte from the table
+
+        raises BlockOutOfRangeError if the block number is higher
+               than the allocation table max size
+        """
+        byte, bit = self.get_block_location(block)
+
+        flag = self.bit_flag(bit)
+
+        if table[byte] & flag == flag:
+            table[byte] ^= flag
+
+    def occupy_block(self, block, table):
+        """
+        Occupies a block by setting corresponding bit in map to 1
+        Args:
+            block (int): place of block in table
+            table (list(int)): list from ints in range [-128;127]
+                               each int represents a byte from the table
+
+        raises BlockOutOfRangeError if the block number is higher than
+               the allocation table max size
+        """
+
+        byte, bit = self.get_block_location(block)
+
+        flag = self.bit_flag(bit)
+
+        if table[byte] & flag == 0:
+            table[byte] ^= flag
+
+
+class FileIOMixin:
+    """
+    This class defines basic functionality for handling
+    reading and writing to a binary file
+    """
+
+    def read_bytes(self, place, size):
+        with open(self.file_name, "rb") as f:
+            f.seek(place)
+            info = f.read(size)
+
+        return info
+
+    def write_bytes(self, info, place):
+        with open(self.file_name, "wb") as f:
+            f.seek(place)
+            f.write(info)
+
+
+class DataConverterMixin:
+    """
+    This class provides functionality for converting
+    data (int and string) to and from bytes of fixed size
+    for reading, writing from a binary file
+    """
+
+    # format strings defined in python docs for more information:
+    # https://docs.python.org/3/library/struct.html#format-characters
+    FORMAT_TYPE_SIZES = {
+        'q': 8,  # long long, 8 bytes
+        'i': 4,  # int , 4 bytes
+        'h': 2,  # short, 2 bytes
+        'b': 1,  # signed char , 1 byte
+    }
+
+    INTEGER_OFFSET = {
+        'q': 2 ** 63,
+        'i': 2 ** 31,
+        'h': 2 ** 15,
+        'b': 2 ** 7
+    }
+
+    STRING_END = '~'  # represents end of string (e.g. '\0')
+
+    def encode_string(self, string, size):
+        """
+        Args:
+            string (str): string to be converted
+            size (int): number of chars in string
+        Returns:
+            bytes object
+        """
+        number_of_null_symbols = size - len(string)
+        null_symbols = self.__class__.STRING_END * number_of_null_symbols
+        string += null_symbols
+        return string.encode(encoding='ascii')
+
+    def decode_string(self, bytes_string):
+        string = bytes_string.decode(encoding='ascii')
+        string_end = self.__class__.STRING_END
+        return ''.join(char for char in string if char != string_end)
+
+    def encode_int(self, value, format_type):
+        """
+        Args:
+            value (int): the number to be converted
+            format_type (string): one char representing the format string
+                                  that will be used for the encoding and
+                                  packing of the data in byte object
+        Returns:
+            bytes object
+
+        Raises FormatStringNotSupportedError
+        """
+        if format_type not in self.__class__.FORMAT_TYPE_SIZES:
+            return FormatStringNotSupportedError(format_type)
+
+        return struct.pack(value, format_type)
+
+    def decode_int(self, bytes_int, format_type):
+        if format_type not in self.__class__.FORMAT_TYPE_SIZES:
+            return FormatStringNotSupportedError(format_type)
+        return struct.unpack(format_type, bytes_int)
+
+    def encode_table(self, table=None, format_type='b'):
+        """
+        Args:
+            table (iterable(int)): representing the information
+                                   in the allocation map
+            format_type (string): one char representing the format string
+                                  that will be used for the encoding and
+                                  packing of the data in byte object
+        Returns:
+            bytes: the information given in table ready
+                   to be writen in a binary file
+
+        Raises FormatStringError and FormatStringNotSupportedError
+        """
+
+        element_size = self.__class__.FORMAT_TYPE_SIZES.get(format_type)
+        map_size_in_bytes = self.allocation_map_size / 8
+        number_of_elements = map_size_in_bytes / element_size
+
+        if number_of_elements != len(table):
+            raise FormatStringError
+
+        if not element_size:
+            raise FormatStringNotSupportedError(format_type)
+        if not table:
+            table = self.create_empty_table(number_of_elements)
+
+        format_string = format_type * number_of_elements
+
+        return struct.pack(format_string, *table)
+
+    def decode_table(self, table_byte_string, format_type='b'):
+        format_string = format_type * self.allocation_map_size
+        return struct.unpack(format_string, table_byte_string)
+
+
+class FileManagementMixin:
+    """
+    This class defines functionality for manging data location in files
+    """
+
+    def get_allocation_map_place(self, sequence_number):
+        maps_size_in_bytes = self.allocation_map_size / 8
+        return sequence_number * maps_size_in_bytes * (1 + self.element_size)
+
+    def get_element_place(self, sequence_number):
+        maps_size_in_bytes = self.allocation_map_size / 8
+        offset_form_page_begining = sequence_number % \
+            maps_size_in_bytes * self.element_size
+
+        page_number = sequence_number // maps_size_in_bytes
+        offset_from_file_begining = page_number * \
+            maps_size_in_bytes * (1 + self.element_size)
+
+        return offset_form_page_begining + offset_from_file_begining
+
+
+class StoreBase(GlobalAllocationMapMixin, FileIOMixin,
+                FileManagementMixin, DataConverterMixin):
+    """
+    This class is the base for the node, edge and property store classes.
+    It defines basic file structure and
+    API for saving, retrieving and updating data.
+    """
+
+    def __init__(self, file_name, element_size, allocation_map_size=1024):
+        """
+        Args:
+            file_name (string): path to file store
+            element_size: size of elements
+            allocation_map_size: size of global allocation map
+        """
+        self.file_name = file_name
+        self.element_size = element_size
+        self.allocation_map_size = allocation_map_size
+
+    def encode_element(self, element):
+        """
+        Hook to encode custom elements to be writen to files
+        """
+
+    def decode_element(self, byte_string):
+        """
+        Hook to decode custom elements read from file
+        """
+
+    def add_element(self, element):
+        """
+        Hook to add an element
+        Should return the id of the new element
+        """
+
+    def remove_element(self, element_id):
+        """
+        Hook to remove elements
+        """
+
+    def get_element(self, element_id):
+        """
+        Hook to access elements in store
+        by a given id
+        """

src/storage/graph/__init__.py

@@ -0,0 +1 @@
+{"0": [0, 1, 2, 3, 4, 5, 6, 7], "1": [0, 1, 2, 3, 4, 5, 6], "2": [0, 1, 2, 3, 4, 5, 7], "3": [0, 1, 2, 3, 4, 5], "4": [0, 1, 2, 3, 4, 6, 7], "5": [0, 1, 2, 3, 4, 6], "6": [0, 1, 2, 3, 4, 7], "7": [0, 1, 2, 3, 4], "8": [0, 1, 2, 3, 5, 6, 7], "9": [0, 1, 2, 3, 5, 6], "10": [0, 1, 2, 3, 5, 7], "11": [0, 1, 2, 3, 5], "12": [0, 1, 2, 3, 6, 7], "13": [0, 1, 2, 3, 6], "14": [0, 1, 2, 3, 7], "15": [0, 1, 2, 3], "16": [0, 1, 2, 4, 5, 6, 7], "17": [0, 1, 2, 4, 5, 6], "18": [0, 1, 2, 4, 5, 7], "19": [0, 1, 2, 4, 5], "20": [0, 1, 2, 4, 6, 7], "21": [0, 1, 2, 4, 6], "22": [0, 1, 2, 4, 7], "23": [0, 1, 2, 4], "24": [0, 1, 2, 5, 6, 7], "25": [0, 1, 2, 5, 6], "26": [0, 1, 2, 5, 7], "27": [0, 1, 2, 5], "28": [0, 1, 2, 6, 7], "29": [0, 1, 2, 6], "30": [0, 1, 2, 7], "31": [0, 1, 2], "32": [0, 1, 3, 4, 5, 6, 7], "33": [0, 1, 3, 4, 5, 6], "34": [0, 1, 3, 4, 5, 7], "35": [0, 1, 3, 4, 5], "36": [0, 1, 3, 4, 6, 7], "37": [0, 1, 3, 4, 6], "38": [0, 1, 3, 4, 7], "39": [0, 1, 3, 4], "40": [0, 1, 3, 5, 6, 7], "41": [0, 1, 3, 5, 6], "42": [0, 1, 3, 5, 7], "43": [0, 1, 3, 5], "44": [0, 1, 3, 6, 7], "45": [0, 1, 3, 6], "46": [0, 1, 3, 7], "47": [0, 1, 3], "48": [0, 1, 4, 5, 6, 7], "49": [0, 1, 4, 5, 6], "50": [0, 1, 4, 5, 7], "51": [0, 1, 4, 5], "52": [0, 1, 4, 6, 7], "53": [0, 1, 4, 6], "54": [0, 1, 4, 7], "55": [0, 1, 4], "56": [0, 1, 5, 6, 7], "57": [0, 1, 5, 6], "58": [0, 1, 5, 7], "59": [0, 1, 5], "60": [0, 1, 6, 7], "61": [0, 1, 6], "62": [0, 1, 7], "63": [0, 1], "64": [0, 2, 3, 4, 5, 6, 7], "65": [0, 2, 3, 4, 5, 6], "66": [0, 2, 3, 4, 5, 7], "67": [0, 2, 3, 4, 5], "68": [0, 2, 3, 4, 6, 7], "69": [0, 2, 3, 4, 6], "70": [0, 2, 3, 4, 7], "71": [0, 2, 3, 4], "72": [0, 2, 3, 5, 6, 7], "73": [0, 2, 3, 5, 6], "74": [0, 2, 3, 5, 7], "75": [0, 2, 3, 5], "76": [0, 2, 3, 6, 7], "77": [0, 2, 3, 6], "78": [0, 2, 3, 7], "79": [0, 2, 3], "80": [0, 2, 4, 5, 6, 7], "81": [0, 2, 4, 5, 6], "82": [0, 2, 4, 5, 7], "83": [0, 2, 4, 5], "84": [0, 2, 4, 6, 7], "85": [0, 2, 4, 6], "86": [0, 2, 4, 7], "87": [0, 2, 4], "88": [0, 2, 5, 6, 7], "89": [0, 2, 5, 6], "90": [0, 2, 5, 7], "91": [0, 2, 5], "92": [0, 2, 6, 7], "93": [0, 2, 6], "94": [0, 2, 7], "95": [0, 2], "96": [0, 3, 4, 5, 6, 7], "97": [0, 3, 4, 5, 6], "98": [0, 3, 4, 5, 7], "99": [0, 3, 4, 5], "100": [0, 3, 4, 6, 7], "101": [0, 3, 4, 6], "102": [0, 3, 4, 7], "103": [0, 3, 4], "104": [0, 3, 5, 6, 7], "105": [0, 3, 5, 6], "106": [0, 3, 5, 7], "107": [0, 3, 5], "108": [0, 3, 6, 7], "109": [0, 3, 6], "110": [0, 3, 7], "111": [0, 3], "112": [0, 4, 5, 6, 7], "113": [0, 4, 5, 6], "114": [0, 4, 5, 7], "115": [0, 4, 5], "116": [0, 4, 6, 7], "117": [0, 4, 6], "118": [0, 4, 7], "119": [0, 4], "120": [0, 5, 6, 7], "121": [0, 5, 6], "122": [0, 5, 7], "123": [0, 5], "124": [0, 6, 7], "125": [0, 6], "126": [0, 7], "127": [0], "128": [1, 2, 3, 4, 5, 6, 7], "129": [1, 2, 3, 4, 5, 6], "130": [1, 2, 3, 4, 5, 7], "131": [1, 2, 3, 4, 5], "132": [1, 2, 3, 4, 6, 7], "133": [1, 2, 3, 4, 6], "134": [1, 2, 3, 4, 7], "135": [1, 2, 3, 4], "136": [1, 2, 3, 5, 6, 7], "137": [1, 2, 3, 5, 6], "138": [1, 2, 3, 5, 7], "139": [1, 2, 3, 5], "140": [1, 2, 3, 6, 7], "141": [1, 2, 3, 6], "142": [1, 2, 3, 7], "143": [1, 2, 3], "144": [1, 2, 4, 5, 6, 7], "145": [1, 2, 4, 5, 6], "146": [1, 2, 4, 5, 7], "147": [1, 2, 4, 5], "148": [1, 2, 4, 6, 7], "149": [1, 2, 4, 6], "150": [1, 2, 4, 7], "151": [1, 2, 4], "152": [1, 2, 5, 6, 7], "153": [1, 2, 5, 6], "154": [1, 2, 5, 7], "155": [1, 2, 5], "156": [1, 2, 6, 7], "157": [1, 2, 6], "158": [1, 2, 7], "159": [1, 2], "160": [1, 3, 4, 5, 6, 7], "161": [1, 3, 4, 5, 6], "162": [1, 3, 4, 5, 7], "163": [1, 3, 4, 5], "164": [1, 3, 4, 6, 7], "165": [1, 3, 4, 6], "166": [1, 3, 4, 7], "167": [1, 3, 4], "168": [1, 3, 5, 6, 7], "169": [1, 3, 5, 6], "170": [1, 3, 5, 7], "171": [1, 3, 5], "172": [1, 3, 6, 7], "173": [1, 3, 6], "174": [1, 3, 7], "175": [1, 3], "176": [1, 4, 5, 6, 7], "177": [1, 4, 5, 6], "178": [1, 4, 5, 7], "179": [1, 4, 5], "180": [1, 4, 6, 7], "181": [1, 4, 6], "182": [1, 4, 7], "183": [1, 4], "184": [1, 5, 6, 7], "185": [1, 5, 6], "186": [1, 5, 7], "187": [1, 5], "188": [1, 6, 7], "189": [1, 6], "190": [1, 7], "191": [1], "192": [2, 3, 4, 5, 6, 7], "193": [2, 3, 4, 5, 6], "194": [2, 3, 4, 5, 7], "195": [2, 3, 4, 5], "196": [2, 3, 4, 6, 7], "197": [2, 3, 4, 6], "198": [2, 3, 4, 7], "199": [2, 3, 4], "200": [2, 3, 5, 6, 7], "201": [2, 3, 5, 6], "202": [2, 3, 5, 7], "203": [2, 3, 5], "204": [2, 3, 6, 7], "205": [2, 3, 6], "206": [2, 3, 7], "207": [2, 3], "208": [2, 4, 5, 6, 7], "209": [2, 4, 5, 6], "210": [2, 4, 5, 7], "211": [2, 4, 5], "212": [2, 4, 6, 7], "213": [2, 4, 6], "214": [2, 4, 7], "215": [2, 4], "216": [2, 5, 6, 7], "217": [2, 5, 6], "218": [2, 5, 7], "219": [2, 5], "220": [2, 6, 7], "221": [2, 6], "222": [2, 7], "223": [2], "224": [3, 4, 5, 6, 7], "225": [3, 4, 5, 6], "226": [3, 4, 5, 7], "227": [3, 4, 5], "228": [3, 4, 6, 7], "229": [3, 4, 6], "230": [3, 4, 7], "231": [3, 4], "232": [3, 5, 6, 7], "233": [3, 5, 6], "234": [3, 5, 7], "235": [3, 5], "236": [3, 6, 7], "237": [3, 6], "238": [3, 7], "239": [3], "240": [4, 5, 6, 7], "241": [4, 5, 6], "242": [4, 5, 7], "243": [4, 5], "244": [4, 6, 7], "245": [4, 6], "246": [4, 7], "247": [4], "248": [5, 6, 7], "249": [5, 6], "250": [5, 7], "251": [5], "252": [6, 7], "253": [6], "254": [7], "255": []}