diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 262418473..41153e59a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,16 +179,21 @@ In order to test how a change in docs configuration looks like on ReadTheDocs be Examples can be found in subfolders of [tests/integrations](https://github.com/XRPLF/xrpl-py/tree/main/tests/integration) -## Updating `definitions.json` +## Updating `definitions.json` and models -This should almost always be done using the [`xrpl-codec-gen`](https://github.com/RichardAH/xrpl-codec-gen) script - if the output needs manual intervention afterwards, consider updating the script instead. +To update just the `definitions.json` file: +```bash +poetry run poe definitions https://github.com/XRPLF/rippled/tree/develop +``` + +Any Github branch link or local path to rippled will work here. -1. Clone / pull the latest changes from [rippled](https://github.com/XRPLF/rippled) - Specifically the `develop` branch is usually the right one. -2. Clone / pull the latest changes from [`xrpl-codec-gen`](https://github.com/RichardAH/xrpl-codec-gen) -3. From the `xrpl-codec-gen` tool, follow the steps in the `README.md` to generate a new `definitions.json` file. -4. Replace the `definitions.json` file in the `ripple-binary-codec` with the newly generated file. -5. Verify that the changes make sense by inspection before submitting, as there may be updates required for the `xrpl-codec-gen` tool depending on the latest amendments we're updating to match. +To update the models as well: +```bash +poetry run poe generate https://github.com/XRPLF/rippled/tree/develop +``` +Verify that the changes make sense by inspection before submitting, as there may be updates required for the `xrpl-codec-gen` tool depending on the latest amendments we're updating to match. ## Release process diff --git a/pyproject.toml b/pyproject.toml index 02c9563cc..d8bfeabc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,11 +90,20 @@ precision = 2 test_unit = "coverage run -m unittest discover tests/unit" test_integration = "coverage run -m unittest discover tests/integration" lint = "poetry run flake8 xrpl tests snippets" +definitions = "poetry run python3 tools/generate_definitions.py" [tool.poe.tasks.test] cmd = "python3 -m unittest ${FILE_PATHS}" args = [{ name = "FILE_PATHS", positional = true, multiple = true }] +[tool.poe.tasks.generate] +help = "Generate the models and definitions for a new amendment" +sequence = [ + { cmd = "python3 tools/generate_definitions.py ${FILE_OR_GITHUB_PATH}" }, + { cmd = "python3 tools/generate_tx_models.py ${FILE_OR_GITHUB_PATH}" }, +] +args = [{ name = "FILE_OR_GITHUB_PATH", positional = true, required = true }] + [tool.poe.tasks.test_coverage] sequence = [ { cmd = "coverage run -m unittest discover" }, diff --git a/tools/generate_definitions.py b/tools/generate_definitions.py new file mode 100644 index 000000000..8355559cd --- /dev/null +++ b/tools/generate_definitions.py @@ -0,0 +1,372 @@ +"""Script to generate the definitions.json file from rippled source code.""" + +import os +import re +import sys +from pathlib import Path + +import httpx + +if len(sys.argv) != 2 and len(sys.argv) != 3: + print("Usage: python " + sys.argv[0] + " path/to/rippled [path/to/output/file]") + print( + "Usage: python " + + sys.argv[0] + + " github.com/user/rippled/tree/feature-branch [path/to/output/file]" + ) + sys.exit(1) + +######################################################################## +# Get all necessary files from rippled +######################################################################## + + +def _read_file_from_github(repo: str, filename: str) -> str: + if "tree" not in repo: + repo += "/tree/HEAD" + url = repo.replace("github.com", "raw.githubusercontent.com") + url = url.replace("tree/", "") + url += "/" + filename + if not url.startswith("http"): + url = "https://" + url + try: + response = httpx.get(url) + response.raise_for_status() + return response.text + except httpx.HTTPError as e: + print(f"Error reading {url}: {e}", file=sys.stderr) + sys.exit(1) + + +def _read_file(folder: str, filename: str) -> str: + file_path = Path(folder) / filename + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + return file_path.read_text() + + +func = _read_file_from_github if "github.com" in sys.argv[1] else _read_file + +sfield_h = func(sys.argv[1], "include/xrpl/protocol/SField.h") +sfield_macro_file = func(sys.argv[1], "include/xrpl/protocol/detail/sfields.macro") +ledger_entries_file = func( + sys.argv[1], "include/xrpl/protocol/detail/ledger_entries.macro" +) +ter_h = func(sys.argv[1], "include/xrpl/protocol/TER.h") +transactions_file = func(sys.argv[1], "include/xrpl/protocol/detail/transactions.macro") + + +# Translate from rippled string format to what the binary codecs expect +def _translate(inp: str) -> str: + if re.match(r"^UINT", inp): + if re.search(r"256|160|128|192", inp): + return inp.replace("UINT", "Hash") + else: + return inp.replace("UINT", "UInt") + + non_standard_renames = { + "OBJECT": "STObject", + "ARRAY": "STArray", + "ACCOUNT": "AccountID", + "LEDGERENTRY": "LedgerEntry", + "NOTPRESENT": "NotPresent", + "PATHSET": "PathSet", + "VL": "Blob", + "DIR_NODE": "DirectoryNode", + "PAYCHAN": "PayChannel", + "XCHAIN_BRIDGE": "XChainBridge", + } + if inp in non_standard_renames: + return non_standard_renames[inp] + + parts = inp.split("_") + result = "" + for part in parts: + result += part[0:1].upper() + part[1:].lower() + return result + + +output = "" + + +# add a new line of content to the output +def _add_line(line: str) -> None: + global output + output += line + "\n" + + +# start +_add_line("{") + +######################################################################## +# SField processing +######################################################################## +_add_line(' "FIELDS": [') + +# The ones that are harder to parse directly from SField.cpp +_add_line( + """ [ + "Generic", + { + "isSerialized": false, + "isSigningField": false, + "isVLEncoded": false, + "nth": 0, + "type": "Unknown" + } + ], + [ + "Invalid", + { + "isSerialized": false, + "isSigningField": false, + "isVLEncoded": false, + "nth": -1, + "type": "Unknown" + } + ], + [ + "ObjectEndMarker", + { + "isSerialized": true, + "isSigningField": true, + "isVLEncoded": false, + "nth": 1, + "type": "STObject" + } + ], + [ + "ArrayEndMarker", + { + "isSerialized": true, + "isSigningField": true, + "isVLEncoded": false, + "nth": 1, + "type": "STArray" + } + ], + [ + "taker_gets_funded", + { + "isSerialized": false, + "isSigningField": false, + "isVLEncoded": false, + "nth": 258, + "type": "Amount" + } + ], + [ + "taker_pays_funded", + { + "isSerialized": false, + "isSigningField": false, + "isVLEncoded": false, + "nth": 259, + "type": "Amount" + } + ],""" +) + +# Parse STypes +# Example line: +# STYPE(STI_UINT32, 2) \ +type_hits = re.findall( + r"^ *STYPE\(STI_([^ ]*?) *, *([0-9-]+) *\) *\\?$", sfield_h, re.MULTILINE +) +# name-to-value map - needed for SField processing +type_map = {x[0]: x[1] for x in type_hits} + + +def _is_vl_encoded(t: str) -> str: + if t == "VL" or t == "ACCOUNT" or t == "VECTOR256": + return "true" + return "false" + + +def _is_serialized(t: str, name: str) -> str: + if t == "LEDGERENTRY" or t == "TRANSACTION" or t == "VALIDATION" or t == "METADATA": + return "false" + if name == "hash" or name == "index": + return "false" + return "true" + + +def _is_signing_field(t: str, not_signing_field: str) -> str: + if not_signing_field == "notSigning": + return "false" + if t == "LEDGERENTRY" or t == "TRANSACTION" or t == "VALIDATION" or t == "METADATA": + return "false" + return "true" + + +# Parse SField.cpp for all the SFields and their serialization info +# Example lines: +# TYPED_SFIELD(sfFee, AMOUNT, 8) +# UNTYPED_SFIELD(sfSigners, ARRAY, 3, SField::sMD_Default, SField::notSigning) +sfield_hits = re.findall( + r"^ *[A-Z]*TYPED_SFIELD *\( *sf([^,\n]*),[ \n]*([^, \n]+)[ \n]*,[ \n]*" + r"([0-9]+)(,.*?(notSigning))?", + sfield_macro_file, + re.MULTILINE, +) +sfield_hits += [ + ("hash", "UINT256", "257", "", "notSigning"), + ("index", "UINT256", "258", "", "notSigning"), +] +sfield_hits.sort(key=lambda x: int(type_map[x[1]]) * 2**16 + int(x[2])) +for x in range(len(sfield_hits)): + _add_line(" [") + _add_line(' "' + sfield_hits[x][0] + '",') + _add_line(" {") + _add_line( + ' "isSerialized": ' + + _is_serialized(sfield_hits[x][1], sfield_hits[x][0]) + + "," + ) + _add_line( + ' "isSigningField": ' + + _is_signing_field(sfield_hits[x][1], sfield_hits[x][4]) + + "," + ) + _add_line(' "isVLEncoded": ' + _is_vl_encoded(sfield_hits[x][1]) + ",") + _add_line(' "nth": ' + sfield_hits[x][2] + ",") + _add_line(' "type": "' + _translate(sfield_hits[x][1]) + '"') + _add_line(" }") + _add_line(" ]" + ("," if x < len(sfield_hits) - 1 else "")) + +_add_line(" ],") + +######################################################################## +# Ledger entry type processing +######################################################################## +_add_line(' "LEDGER_ENTRY_TYPES": {') + + +def _unhex(x: str) -> str: + if x[0:2] == "0x": + return str(int(x, 16)) + return x + + +# Parse ledger entries +# Example line: +# LEDGER_ENTRY(ltNFTOKEN_OFFER, 0x0037, NFTokenOffer, nft_offer, ({ +lt_hits = re.findall( + r"^ *LEDGER_ENTRY[A-Z_]*\(lt[A-Z_]+ *, *([xX0-9a-fA-F]+) *, *([^,]+), *([^,]+), " + r"\({$", + ledger_entries_file, + re.MULTILINE, +) +lt_hits.append(("-1", "Invalid")) +lt_hits.sort(key=lambda x: x[1]) +for x in range(len(lt_hits)): + _add_line( + ' "' + + lt_hits[x][1] + + '": ' + + _unhex(lt_hits[x][0]) + + ("," if x < len(lt_hits) - 1 else "") + ) +_add_line(" },") + +######################################################################## +# TER code processing +######################################################################## +_add_line(' "TRANSACTION_RESULTS": {') +ter_h = str(ter_h).replace("[[maybe_unused]]", "") + +# Parse TER codes +ter_code_hits = re.findall( + r"^ *((tel|tem|tef|ter|tes|tec)[A-Z_]+)( *= *([0-9-]+))? *,? *(\/\/[^\n]*)?$", + ter_h, + re.MULTILINE, +) +ter_codes = [] +upto = -1 + +# Get the exact values of the TER codes and sort them +for x in range(len(ter_code_hits)): + if ter_code_hits[x][3] != "": + upto = int(ter_code_hits[x][3]) + ter_codes.append((ter_code_hits[x][0], upto)) + + upto += 1 +ter_codes.sort(key=lambda x: x[0]) + +current_type = "" +for x in range(len(ter_codes)): + # print newline between the different code types + if current_type == "": + current_type = ter_codes[x][0][:3] + elif current_type != ter_codes[x][0][:3]: + _add_line("") + current_type = ter_codes[x][0][:3] + + _add_line( + ' "' + + ter_codes[x][0] + + '": ' + + str(ter_codes[x][1]) + + ("," if x < len(ter_codes) - 1 else "") + ) + +_add_line(" },") + +######################################################################## +# Transaction type processing +######################################################################## +_add_line(' "TRANSACTION_TYPES": {') + +# Parse transaction types +# Example line: +# TRANSACTION(ttCHECK_CREATE, 16, CheckCreate, ({ +tx_hits = re.findall( + r"^ *TRANSACTION\(tt[A-Z_]+ *,* ([0-9]+) *, *([A-Za-z]+).*$", + transactions_file, + re.MULTILINE, +) +tx_hits.append(("-1", "Invalid")) +tx_hits.sort(key=lambda x: x[1]) + +for x in range(len(tx_hits)): + _add_line( + ' "' + + tx_hits[x][1] + + '": ' + + tx_hits[x][0] + + ("," if x < len(tx_hits) - 1 else "") + ) + +_add_line(" },") + +######################################################################## +# Serialized type processing +######################################################################## +_add_line(' "TYPES": {') + +type_hits.append(("DONE", "-1")) +type_hits.sort(key=lambda x: _translate(x[0])) +for x in range(len(type_hits)): + _add_line( + ' "' + + _translate(type_hits[x][0]) + + '": ' + + type_hits[x][1] + + ("," if x < len(type_hits) - 1 else "") + ) + +_add_line(" }") +_add_line("}") + + +if len(sys.argv) == 3: + output_file = sys.argv[2] +else: + output_file = os.path.join( + os.path.dirname(__file__), + "../xrpl/core/binarycodec/definitions/definitions.json", + ) + +with open(output_file, "w") as f: + f.write(output) +print("File written successfully to " + output_file) diff --git a/tools/generate_tx_models.py b/tools/generate_tx_models.py index 28b0dcbef..d13e6ea01 100644 --- a/tools/generate_tx_models.py +++ b/tools/generate_tx_models.py @@ -5,13 +5,25 @@ import sys from typing import Dict, List, Tuple +import httpx + from xrpl.models.base_model import _key_to_json from xrpl.models.transactions.types.pseudo_transaction_type import PseudoTransactionType from xrpl.models.transactions.types.transaction_type import TransactionType -def _read_file(filename: str) -> str: - with open(filename) as f: +def _read_file_from_github(repo: str, filename: str) -> str: + url = repo.replace("github.com", "raw.githubusercontent.com") + url = url.replace("tree", "refs/heads") + url += "/" + filename + if not url.startswith("http"): + url = "https://" + url + response = httpx.get(url) + return response.text + + +def _read_file(folder: str, filename: str) -> str: + with open(os.path.join(folder, filename), "r") as f: return f.read() @@ -25,23 +37,25 @@ def _format_tx_format(raw_tx_format: str) -> List[Tuple[str, ...]]: def _parse_rippled_source( folder: str, ) -> Tuple[Dict[str, List[str]], Dict[str, List[Tuple[str, ...]]]]: + func = _read_file_from_github if "github.com" in sys.argv[1] else _read_file # Get SFields - sfield_cpp = _read_file(os.path.join(folder, "src/ripple/protocol/impl/SField.cpp")) + sfield_cpp = func(folder, "include/xrpl/protocol/detail/sfields.macro") sfield_hits = re.findall( - r'^ *CONSTRUCT_[^\_]+_SFIELD *\( *[^,\n]*,[ \n]*"([^\"\n ]+)"[ \n]*,[ \n]*' - + r"([^, \n]+)[ \n]*,[ \n]*([0-9]+)(,.*?(notSigning))?", + ( + r"^ *[A-Z]*TYPED_SFIELD *\( *sf([^,\n]*),[ \n]*([^, \n]+)[ \n]*,[ \n]*" + r"([0-9]+)(,.*?(notSigning))?" + ), sfield_cpp, re.MULTILINE, ) sfields = {hit[0]: hit[1:] for hit in sfield_hits} # Get TxFormats - tx_formats_cpp = _read_file( - os.path.join(folder, "src/ripple/protocol/impl/TxFormats.cpp") - ) + tx_formats_cpp = func(folder, "include/xrpl/protocol/detail/transactions.macro") tx_formats_hits = re.findall( - r"^ *add\(jss::([^\"\n, ]+),[ \n]*tt[A-Z_]+,[ \n]*{[ \n]*(({sf[A-Za-z0-9]+, " - + r"soe(OPTIONAL|REQUIRED|DEFAULT)},[ \n]+)*)},[ \n]*[pseudocC]+ommonFields\);", + r"^ *TRANSACTION\(tt[A-Z_]+ *,* [0-9]+ *, *([A-Za-z]+)[ \n]*,[ \n]*\({[ \n]*" + + r"(({sf[A-Za-z0-9]+, soe(OPTIONAL|REQUIRED|DEFAULT)" + + r"(, soeMPT(None|Supported|NotSupported))?},[ \n]+)*)}\)\)$", tx_formats_cpp, re.MULTILINE, ) @@ -57,6 +71,7 @@ def _parse_rippled_source( "UINT64": "Union[int, str]", "UINT128": "str", "UINT160": "str", + "UINT192": "str", "UINT256": "str", "AMOUNT": "Amount", "VL": "str", @@ -130,7 +145,7 @@ def _generate_param_line(param: str, is_required: bool) -> str: param_lines.sort(key=lambda x: "REQUIRED" not in x) params = "\n".join(param_lines) model = f"""@require_kwargs_on_init -@dataclass(frozen=True, **KW_ONLY_DATACLASS) +@dataclass(frozen=True, **KW_ONLY_DATACLASS) class {tx}(Transaction): \"\"\"Represents a {tx} transaction.\"\"\" diff --git a/xrpl/core/binarycodec/definitions/definitions.json b/xrpl/core/binarycodec/definitions/definitions.json index 92e90c8e6..bc86c2c19 100644 --- a/xrpl/core/binarycodec/definitions/definitions.json +++ b/xrpl/core/binarycodec/definitions/definitions.json @@ -913,20 +913,20 @@ [ "IssuerNode", { - "nth": 27, - "isVLEncoded": false, "isSerialized": true, "isSigningField": true, + "isVLEncoded": false, + "nth": 27, "type": "UInt64" } ], [ "SubjectNode", { - "nth": 28, - "isVLEncoded": false, "isSerialized": true, "isSigningField": true, + "isVLEncoded": false, + "nth": 28, "type": "UInt64" } ], @@ -1253,10 +1253,10 @@ [ "DomainID", { - "nth": 34, - "isVLEncoded": false, "isSerialized": true, "isSigningField": true, + "isVLEncoded": false, + "nth": 34, "type": "Hash256" } ], @@ -1843,10 +1843,10 @@ [ "CredentialType", { - "nth": 31, - "isVLEncoded": true, "isSerialized": true, "isSigningField": true, + "isVLEncoded": true, + "nth": 31, "type": "Blob" } ], @@ -2023,13 +2023,23 @@ [ "Subject", { - "nth": 24, - "isVLEncoded": true, "isSerialized": true, "isSigningField": true, + "isVLEncoded": true, + "nth": 24, "type": "AccountID" } ], + [ + "Number", + { + "isSerialized": true, + "isSigningField": true, + "isVLEncoded": false, + "nth": 1, + "type": "Number" + } + ], [ "TransactionMetaData", { @@ -2323,10 +2333,10 @@ [ "Credential", { - "nth": 33, - "isVLEncoded": false, "isSerialized": true, "isSigningField": true, + "isVLEncoded": false, + "nth": 33, "type": "STObject" } ], @@ -2523,29 +2533,30 @@ [ "AuthorizeCredentials", { - "nth": 26, - "isVLEncoded": false, "isSerialized": true, "isSigningField": true, + "isVLEncoded": false, + "nth": 26, "type": "STArray" } ], [ "UnauthorizeCredentials", { - "nth": 27, - "isVLEncoded": false, "isSerialized": true, "isSigningField": true, + "isVLEncoded": false, + "nth": 27, "type": "STArray" } ], [ - "AcceptedCredentials", { - "nth": 28, - "isVLEncoded": false, + "AcceptedCredentials", + { "isSerialized": true, "isSigningField": true, + "isVLEncoded": false, + "nth": 28, "type": "STArray" } ], @@ -2732,10 +2743,10 @@ [ "CredentialIDs", { - "nth": 5, - "isVLEncoded": true, "isSerialized": true, "isSigningField": true, + "isVLEncoded": true, + "nth": 5, "type": "Vector256" } ], @@ -2866,6 +2877,7 @@ "Amendments": 102, "Bridge": 105, "Check": 67, + "Credential": 129, "DID": 73, "DepositPreauth": 112, "DirectoryNode": 100, @@ -2880,7 +2892,6 @@ "NegativeUNL": 78, "Offer": 111, "Oracle": 128, - "Credential": 129, "PayChannel": 120, "PermissionedDomain": 130, "RippleState": 114, @@ -2910,6 +2921,7 @@ "tecFAILED_PROCESSING": 105, "tecFROZEN": 137, "tecHAS_OBLIGATIONS": 151, + "tecHOOK_REJECTED": 153, "tecINCOMPLETE": 169, "tecINSUFFICIENT_FUNDS": 159, "tecINSUFFICIENT_PAYMENT": 161, @@ -2968,6 +2980,7 @@ "tecXCHAIN_SELF_COMMIT": 184, "tecXCHAIN_SENDING_ACCOUNT_MISMATCH": 179, "tecXCHAIN_WRONG_CHAIN": 176, + "tefALREADY": -198, "tefBAD_ADD_AUTH": -197, "tefBAD_AUTH": -196, @@ -2990,6 +3003,7 @@ "tefPAST_SEQ": -190, "tefTOO_BIG": -181, "tefWRONG_PRIOR": -189, + "telBAD_DOMAIN": -398, "telBAD_PATH_COUNT": -397, "telBAD_PUBLIC_KEY": -396, @@ -3007,6 +3021,7 @@ "telNO_DST_PARTIAL": -393, "telREQUIRES_NETWORK_ID": -385, "telWRONG_NETWORK": -386, + "temARRAY_EMPTY": -253, "temARRAY_TOO_LARGE": -252, "temBAD_AMM_TOKENS": -261, @@ -3056,6 +3071,7 @@ "temXCHAIN_BRIDGE_BAD_REWARD_AMOUNT": -255, "temXCHAIN_BRIDGE_NONDOOR_OWNER": -257, "temXCHAIN_EQUAL_DOOR_ACCOUNTS": -260, + "terFUNDS_SPENT": -98, "terINSUF_FEE_B": -97, "terLAST": -91, @@ -3069,6 +3085,7 @@ "terPRE_TICKET": -88, "terQUEUED": -89, "terRETRY": -99, + "tesSUCCESS": 0 }, "TRANSACTION_TYPES": { @@ -3085,8 +3102,8 @@ "CheckCash": 17, "CheckCreate": 16, "Clawback": 30, - "CredentialCreate": 58, "CredentialAccept": 59, + "CredentialCreate": 58, "CredentialDelete": 60, "DIDDelete": 50, "DIDSet": 49, @@ -3115,8 +3132,8 @@ "PaymentChannelClaim": 15, "PaymentChannelCreate": 13, "PaymentChannelFund": 14, - "PermissionedDomainSet": 62, "PermissionedDomainDelete": 63, + "PermissionedDomainSet": 62, "SetFee": 101, "SetRegularKey": 5, "SignerListSet": 12, @@ -3146,6 +3163,7 @@ "LedgerEntry": 10002, "Metadata": 10004, "NotPresent": 0, + "Number": 9, "PathSet": 18, "STArray": 15, "STObject": 14,