Optimize the process of program deployment and upgrade

Author: mohanson

README.md

@@ -142,7 +142,7 @@ $ python example/program.py --prikey 0x1 --action update --addr 6B7KVuUQ42x8SagF
 
 $ python example/program.py --prikey 0x1 --action call --addr 6B7KVuUQ42x8SagFuFaoiV9jWTSic3Qd771kNrmGwoBG
 # Program 6B7KVuUQ42x8SagFuFaoiV9jWTSic3Qd771kNrmGwoBG invoke [1]
-# Program log: Hello, Update!
+# Program log: Hello, Solana! Hello Update!
 # Program log: Our program's Program ID: 6B7KVuUQ42x8SagFuFaoiV9jWTSic3Qd771kNrmGwoBG
 # Program 6B7KVuUQ42x8SagFuFaoiV9jWTSic3Qd771kNrmGwoBG consumed 11850 of 200000 compute units
 # Program 6B7KVuUQ42x8SagFuFaoiV9jWTSic3Qd771kNrmGwoBG success

example/program.py

@@ -45,7 +45,7 @@
     print('Program', pubkey, 'create')
 
 if args.action == 'update':
-    data = bytearray(pathlib.Path('res/hello_solana_program.so.2').read_bytes())
+    data = bytearray(pathlib.Path('res/hello_update_program.so').read_bytes())
     pubkey = pxsol.core.PubKey.base58_decode(args.addr)
     user.program_update(pubkey, data)
     print('Program', pubkey, 'update')

pxsol/program.py

@@ -163,7 +163,7 @@ def close(cls) -> bytearray:
     @classmethod
     def extend_program(cls, size: int) -> bytearray:
         # Extend a program's program data account by the specified number of bytes. Only upgradeable program's can be
-        # extended. Account references:
+        # extended. Extend program was superseded by extend program checked. Account references:
         # 0. -w the program data account.
         # 1. -w the program data account's associated program account.
         # 2. -r system program, optional, used to transfer lamports from the payer to the program data account.

pxsol/rpc.py

@@ -52,6 +52,7 @@ def step() -> None:
     # Waiting for at least one new block.
     data = get_block_height({})
     for _ in itertools.repeat(0):
+        time.sleep(max(0.5, 1 / pxsol.config.current.rpc.qps))
         if get_block_height({}) != data:
             break
 

pxsol/wallet.py

@@ -22,37 +22,23 @@ def json(self) -> typing.Dict:
             'pubkey': self.pubkey.base58(),
         }
 
-    def program_buffer_closed(self, program_buffer: pxsol.core.PubKey) -> None:
-        # Close a buffer account. This method is used to withdraw all lamports when the buffer account is no longer in
-        # use due to unexpected errors.
-        rq = pxsol.core.Requisition(pxsol.program.LoaderUpgradeable.pubkey, [], bytearray())
-        rq.account.append(pxsol.core.AccountMeta(program_buffer, 1))
-        rq.account.append(pxsol.core.AccountMeta(self.pubkey, 1))
-        rq.account.append(pxsol.core.AccountMeta(self.pubkey, 2))
-        rq.data = pxsol.program.LoaderUpgradeable.close()
-        tx = pxsol.core.Transaction.requisition_decode(self.pubkey, [rq])
-        tx.message.recent_blockhash = pxsol.base58.decode(pxsol.rpc.get_latest_blockhash({})['blockhash'])
-        tx.sign([self.prikey])
-        txid = pxsol.rpc.send_transaction(base64.b64encode(tx.serialize()).decode(), {})
-        pxsol.rpc.wait([txid])
-
-    def program_buffer_create(self, bincode: bytearray) -> pxsol.core.PubKey:
+    def program_buffer(self, bincode: bytearray) -> pxsol.core.PubKey:
         # Writes a program into a buffer account. The buffer account is randomly generated, and its public key serves
         # as the function's return value.
         tempory_prikey = pxsol.core.PriKey.random()
         program_buffer = tempory_prikey.pubkey()
+        pxsol.log.debugln(f'pxsol: program buffer prikey={tempory_prikey}')
+        pxsol.log.debugln(f'pxsol: program buffer pubkey={program_buffer}')
+        account_length = pxsol.program.LoaderUpgradeable.size_program_buffer + len(bincode)
         # Sends a transaction which creates a buffer account large enough for the byte-code being deployed. It also
         # invokes the initialize buffer instruction to set the buffer authority to restrict writes to the deployer's
         # chosen address.
         r0 = pxsol.core.Requisition(pxsol.program.System.pubkey, [], bytearray())
         r0.account.append(pxsol.core.AccountMeta(self.pubkey, 3))
         r0.account.append(pxsol.core.AccountMeta(program_buffer, 3))
         r0.data = pxsol.program.System.create_account(
-            pxsol.rpc.get_minimum_balance_for_rent_exemption(
-                pxsol.program.LoaderUpgradeable.size_program_data + len(bincode),
-                {},
-            ),
-            pxsol.program.LoaderUpgradeable.size_program_buffer + len(bincode),
+            pxsol.rpc.get_minimum_balance_for_rent_exemption(account_length, {}),
+            account_length,
             pxsol.program.LoaderUpgradeable.pubkey,
         )
         r1 = pxsol.core.Requisition(pxsol.program.LoaderUpgradeable.pubkey, [], bytearray())
@@ -100,10 +86,12 @@ def program_closed(self, program: pxsol.core.PubKey) -> None:
 
     def program_deploy(self, bincode: bytearray) -> pxsol.core.PubKey:
         # Deploying a program on solana, returns the program's public key.
+        program_buffer = self.program_buffer(bincode)
         tempory_prikey = pxsol.core.PriKey.random()
-        program_buffer = self.program_buffer_create(bincode)
         program = tempory_prikey.pubkey()
         program_data = pxsol.program.LoaderUpgradeable.pubkey.derive_pda(program.p)
+        pxsol.log.debugln(f'pxsol: program prikey={tempory_prikey}')
+        pxsol.log.debugln(f'pxsol: program pubkey={program}')
         # Deploy with max data len.
         r0 = pxsol.core.Requisition(pxsol.program.System.pubkey, [], bytearray())
         r0.account.append(pxsol.core.AccountMeta(self.pubkey, 3))
@@ -122,19 +110,37 @@ def program_deploy(self, bincode: bytearray) -> pxsol.core.PubKey:
         r1.account.append(pxsol.core.AccountMeta(pxsol.program.SysvarClock.pubkey, 0))
         r1.account.append(pxsol.core.AccountMeta(pxsol.program.System.pubkey, 0))
         r1.account.append(pxsol.core.AccountMeta(self.pubkey, 2))
-        r1.data = pxsol.program.LoaderUpgradeable.deploy_with_max_data_len(len(bincode) * 2)
+        r1.data = pxsol.program.LoaderUpgradeable.deploy_with_max_data_len(len(bincode))
         tx = pxsol.core.Transaction.requisition_decode(self.pubkey, [r0, r1])
         tx.message.recent_blockhash = pxsol.base58.decode(pxsol.rpc.get_latest_blockhash({})['blockhash'])
         tx.sign([self.prikey, tempory_prikey])
         txid = pxsol.rpc.send_transaction(base64.b64encode(tx.serialize()).decode(), {})
         pxsol.rpc.wait([txid])
-        pxsol.rpc.step()
         return program
 
     def program_update(self, program: pxsol.core.PubKey, bincode: bytearray) -> None:
         # Updating an existing solana program by new program data and the same program id.
-        program_buffer = self.program_buffer_create(bincode)
+        program_buffer = self.program_buffer(bincode)
         program_data = pxsol.program.LoaderUpgradeable.pubkey.derive_pda(program.p)
+        # Check the existing program data account size. If the new program data is larger than the existing one,
+        # extend the program data account first.
+        program_data_info = pxsol.rpc.get_account_info(program_data.base58(), {})
+        assert len(base64.b64decode(program_data_info['data'][0])) == program_data_info['space']
+        addi = pxsol.program.LoaderUpgradeable.size_program_data + len(bincode) - program_data_info['space']
+        if addi > 0:
+            pxsol.log.debugln(f'pxsol: extend program data addi={addi}')
+            rq = pxsol.core.Requisition(pxsol.program.LoaderUpgradeable.pubkey, [], bytearray())
+            rq.account.append(pxsol.core.AccountMeta(program_data, 1))
+            rq.account.append(pxsol.core.AccountMeta(program, 1))
+            rq.account.append(pxsol.core.AccountMeta(self.pubkey, 2))
+            rq.account.append(pxsol.core.AccountMeta(pxsol.program.System.pubkey, 0))
+            rq.account.append(pxsol.core.AccountMeta(self.pubkey, 3))
+            rq.data = pxsol.program.LoaderUpgradeable.extend_program_checked(addi)
+            tx = pxsol.core.Transaction.requisition_decode(self.pubkey, [rq])
+            tx.message.recent_blockhash = pxsol.base58.decode(pxsol.rpc.get_latest_blockhash({})['blockhash'])
+            tx.sign([self.prikey])
+            txid = pxsol.rpc.send_transaction(base64.b64encode(tx.serialize()).decode(), {})
+            pxsol.rpc.wait([txid])
         rq = pxsol.core.Requisition(pxsol.program.LoaderUpgradeable.pubkey, [], bytearray())
         rq.account.append(pxsol.core.AccountMeta(program_data, 1))
         rq.account.append(pxsol.core.AccountMeta(program, 1))
@@ -149,7 +155,6 @@ def program_update(self, program: pxsol.core.PubKey, bincode: bytearray) -> None
         tx.sign([self.prikey])
         txid = pxsol.rpc.send_transaction(base64.b64encode(tx.serialize()).decode(), {})
         pxsol.rpc.wait([txid])
-        pxsol.rpc.step()
 
     def sol_balance(self) -> int:
         # Returns the lamport balance of the account.

res/hello_solana_program.md

@@ -4,6 +4,6 @@
 
 ```sh
 $ git clone https://github.com/solana-developers/program-examples
-$ cd program-examples/basics/hello-solana/native
-$ cargo build-bpf --manifest-path=./program/Cargo.toml --bpf-out-dir=./program/target/so
+$ cd program-examples/basics/hello-solana/pinocchio
+$ cargo build-sbf --manifest-path=./program/Cargo.toml --sbf-out-dir=./program/target/so
 ```

res/hello_solana_program.so

@@ -2,23 +2,31 @@
 import pathlib
 import pxsol
 import random
+import typing
 
 
 def test_program():
     user = pxsol.wallet.Wallet(pxsol.core.PriKey.int_decode(1))
-    program_data = bytearray(pathlib.Path('res/hello_solana_program.so').read_bytes())
-    program_pubkey = user.program_deploy(program_data)
-    program_data_update = bytearray(pathlib.Path('res/hello_solana_program.so.2').read_bytes())
-    user.program_update(program_pubkey, program_data_update)
-    pxsol.rpc.step()
-    user.program_closed(program_pubkey)
+    program_hello_solana = bytearray(pathlib.Path('res/hello_solana_program.so').read_bytes())
+    program_hello_update = bytearray(pathlib.Path('res/hello_update_program.so').read_bytes())
 
+    def call(program_pubkey: pxsol.core.PubKey) -> typing.List[str]:
+        rq = pxsol.core.Requisition(program_pubkey, [], bytearray())
+        tx = pxsol.core.Transaction.requisition_decode(user.pubkey, [rq])
+        tx.message.recent_blockhash = pxsol.base58.decode(pxsol.rpc.get_latest_blockhash({})['blockhash'])
+        tx.sign([user.prikey])
+        txid = pxsol.rpc.send_transaction(base64.b64encode(tx.serialize()).decode(), {})
+        pxsol.rpc.wait([txid])
+        r = pxsol.rpc.get_transaction(txid, {})
+        return r['meta']['logMessages']
 
-def test_program_buffer():
-    user = pxsol.wallet.Wallet(pxsol.core.PriKey.int_decode(1))
-    pubkey = user.program_buffer_create(bytearray(pathlib.Path('res/hello_solana_program.so').read_bytes()))
-    pxsol.rpc.step()
-    user.program_buffer_closed(pubkey)
+    program_pubkey = user.program_deploy(program_hello_solana)
+    assert call(program_pubkey)[1] == 'Program log: Hello, Solana!'
+    user.program_update(program_pubkey, program_hello_update)
+    assert call(program_pubkey)[1] == 'Program log: Hello, Solana! Hello Update!'
+    user.program_update(program_pubkey, program_hello_solana)
+    assert call(program_pubkey)[1] == 'Program log: Hello, Solana!'
+    user.program_closed(program_pubkey)
 
 
 def test_sol_transfer():