From 9e4968cf679c4ec0a799ecd5f8e401618cf08150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 08:33:54 +0200 Subject: [PATCH 01/17] Add special token modification capability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To be able to fix/amend special tokens in a GGUF let's add two new arguments: * `--special-token ` where `` can be bos, eos, prefix, middle, etc. while `` is the token value, f.ex. `"<｜fim▁begin｜>"` * `--special-token-by-id ` where `` is the ID of the token, f.ex. 32006 So, in order to f.ex. add fill-in-middle tokens to a GGUF you would do the following: ```bash python3 gguf-new-metadata.py input.gguf output.gguf --special-token prefix "<｜fim▁begin｜>" --special-token middle "<｜fim▁hole｜>" --special-token suffix "<｜fim▁end｜>" ``` --- gguf-py/scripts/gguf-new-metadata.py | 74 ++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 3444ab41802c5..7403e90eb0007 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -8,6 +8,7 @@ import numpy as np from typing import Any, Mapping, Sequence +from dataclasses import dataclass # Necessary to load the local gguf package if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): @@ -18,6 +19,12 @@ logger = logging.getLogger("gguf-new-metadata") +@dataclass +class MetadataDetails: + type: gguf.GGUFValueType + value: Any + description: str = '' + def get_byteorder(reader: gguf.GGUFReader) -> gguf.GGUFEndian: if np.uint32(1) == np.uint32(1).newbyteorder("<"): # Host is little endian @@ -59,7 +66,16 @@ def get_field_data(reader: gguf.GGUFReader, key: str) -> Any: return decode_field(field) -def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, str], remove_metadata: Sequence[str]) -> None: +def find_token(token_list: Sequence[int], token: str) -> Sequence[int]: + token_ids = [index for index, value in enumerate(token_list) if value == token] + + if len(token_ids) == 0: + raise LookupError(f'Unable to find "{token}" in token list!') + + return token_ids + + +def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: for field in reader.fields.values(): # Suppress virtual fields and fields written by GGUFWriter if field.name == gguf.Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'): @@ -75,29 +91,28 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new logger.debug(f'Removing {field.name}') continue - old_val = decode_field(field) + old_val = MetadataDetails(field.types[0], decode_field(field)) val = new_metadata.get(field.name, old_val) if field.name in new_metadata: - logger.debug(f'Modifying {field.name}: "{old_val}" -> "{val}"') + logger.debug(f'Modifying {field.name}: "{old_val.value}" -> "{val.value}" {val.description}') del new_metadata[field.name] - elif val is not None: + elif val.value is not None: logger.debug(f'Copying {field.name}') - if val is not None: + if val.value is not None: writer.add_key(field.name) - writer.add_val(val, field.types[0]) + writer.add_val(val.value, val.type) if gguf.Keys.Tokenizer.CHAT_TEMPLATE in new_metadata: logger.debug('Adding chat template(s)') - writer.add_chat_template(new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE]) + writer.add_chat_template(new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE].value) del new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] - # TODO: Support other types than string? for key, val in new_metadata.items(): - logger.debug(f'Adding {key}: {val}') + logger.debug(f'Adding {key}: "{val.value}" {val.description}') writer.add_key(key) - writer.add_val(val, gguf.GGUFValueType.STRING) + writer.add_val(val.value, val.type) for tensor in reader.tensors: # Dimensions are written in reverse order, so flip them first @@ -115,6 +130,9 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new def main() -> None: + tokenizer_metadata = (getattr(gguf.Keys.Tokenizer, n) for n in gguf.Keys.Tokenizer.__dict__.keys() if not n.startswith('_')) + token_names = dict((n.split('.')[-1][:-len('_token_id')], n) for n in tokenizer_metadata if n.endswith('_token_id')) + parser = argparse.ArgumentParser(description="Make a copy of a GGUF file with new metadata") parser.add_argument("input", type=Path, help="GGUF format model input filename") parser.add_argument("output", type=Path, help="GGUF format model output filename") @@ -123,6 +141,8 @@ def main() -> None: parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)") parser.add_argument("--chat-template-config", type=Path, help="Config file (tokenizer_config.json) containing chat template(s)") parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model") + parser.add_argument("--special-token", action="append", type=str, help="Special token by value", nargs=2, metavar=(' | '.join(token_names.keys()), '""')) + parser.add_argument("--special-token-by-id", action="append", type=str, help="Special token by id", nargs=2, metavar=(' | '.join(token_names.keys()), '0')) parser.add_argument("--force", action="store_true", help="Bypass warnings without confirmation") parser.add_argument("--verbose", action="store_true", help="Increase output verbosity") args = parser.parse_args(None if len(sys.argv) > 2 else ["--help"]) @@ -133,20 +153,20 @@ def main() -> None: remove_metadata = args.remove_metadata or [] if args.general_name: - new_metadata[gguf.Keys.General.NAME] = args.general_name + new_metadata[gguf.Keys.General.NAME] = MetadataDetails(gguf.GGUFValueType.STRING, args.general_name) if args.general_description: - new_metadata[gguf.Keys.General.DESCRIPTION] = args.general_description + new_metadata[gguf.Keys.General.DESCRIPTION] = MetadataDetails(gguf.GGUFValueType.STRING, args.general_description) if args.chat_template: - new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = json.loads(args.chat_template) if args.chat_template.startswith('[') else args.chat_template + new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, json.loads(args.chat_template) if args.chat_template.startswith('[') else args.chat_template) if args.chat_template_config: with open(args.chat_template_config, 'r') as fp: config = json.load(fp) template = config.get('chat_template') if template: - new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = template + new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, template) if remove_metadata: logger.warning('*** Warning *** Warning *** Warning **') @@ -166,6 +186,32 @@ def main() -> None: arch = get_field_data(reader, gguf.Keys.General.ARCHITECTURE) endianess = get_byteorder(reader) + token_list = get_field_data(reader, gguf.Keys.Tokenizer.LIST) or [] + + for name, token in args.special_token or []: + if name not in token_names: + logger.warning(f'Unknown special token "{name}", ignoring...') + else: + ids = find_token(token_list, token) + new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, ids[0], f'= {token}') + + if len(ids) > 1: + logger.warning(f'Multiple "{token}" tokens found, choosing ID {ids[0]}, use --special-token-by-id if you want another:') + logger.warning(', '.join(ids)) + + for name, id_string in args.special_token_by_id or []: + if name not in token_names: + logger.warning(f'Unknown special token "{name}", ignoring...') + elif not id_string.isdecimal(): + logger.warning(f'Token ID "{id_string}" is not a valid ID, ignoring...') + else: + id_int = int(id_string) + + if id_int >= 0 and id_int < len(token_list): + new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, id_int, f'= {token_list[id_int]}') + else: + logger.warning(f'Token ID {id_int} is not within token list, ignoring...') + if os.path.isfile(args.output) and not args.force: logger.warning('*** Warning *** Warning *** Warning **') logger.warning(f'* The "{args.output}" GGUF file already exists, it will be overwritten!') From 8d36967a2cd01a210093f56bb4704c162fcd6793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 08:45:33 +0200 Subject: [PATCH 02/17] improve help text --- gguf-py/scripts/gguf-new-metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 7403e90eb0007..3f743864c3c31 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -136,11 +136,11 @@ def main() -> None: parser = argparse.ArgumentParser(description="Make a copy of a GGUF file with new metadata") parser.add_argument("input", type=Path, help="GGUF format model input filename") parser.add_argument("output", type=Path, help="GGUF format model output filename") - parser.add_argument("--general-name", type=str, help="The models general.name") - parser.add_argument("--general-description", type=str, help="The models general.description") - parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)") - parser.add_argument("--chat-template-config", type=Path, help="Config file (tokenizer_config.json) containing chat template(s)") - parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model") + parser.add_argument("--general-name", type=str, help="The models general.name", metavar='"name"') + parser.add_argument("--general-description", type=str, help="The models general.description", metavar='"Description ..."') + parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)", metavar='"{% ... %} ..."') + parser.add_argument("--chat-template-config", type=Path, help="Config file containing chat template(s)", metavar='tokenizer_config.json') + parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model", metavar='general.url') parser.add_argument("--special-token", action="append", type=str, help="Special token by value", nargs=2, metavar=(' | '.join(token_names.keys()), '""')) parser.add_argument("--special-token-by-id", action="append", type=str, help="Special token by id", nargs=2, metavar=(' | '.join(token_names.keys()), '0')) parser.add_argument("--force", action="store_true", help="Bypass warnings without confirmation") From c4e6f6f4b932c315c61497b8382b6b6afaa99dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 08:48:39 +0200 Subject: [PATCH 03/17] flake-- --- gguf-py/scripts/gguf-new-metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 3f743864c3c31..b7d46b682806a 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -25,6 +25,7 @@ class MetadataDetails: value: Any description: str = '' + def get_byteorder(reader: gguf.GGUFReader) -> gguf.GGUFEndian: if np.uint32(1) == np.uint32(1).newbyteorder("<"): # Host is little endian @@ -67,12 +68,12 @@ def get_field_data(reader: gguf.GGUFReader, key: str) -> Any: def find_token(token_list: Sequence[int], token: str) -> Sequence[int]: - token_ids = [index for index, value in enumerate(token_list) if value == token] + token_ids = [index for index, value in enumerate(token_list) if value == token] - if len(token_ids) == 0: - raise LookupError(f'Unable to find "{token}" in token list!') + if len(token_ids) == 0: + raise LookupError(f'Unable to find "{token}" in token list!') - return token_ids + return token_ids def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: From a2410b6cb26a080992a783be503df416f14919a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 09:34:01 +0200 Subject: [PATCH 04/17] fix multiple tokens warning --- gguf-py/scripts/gguf-new-metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index b7d46b682806a..3963d13dd48f1 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -198,7 +198,7 @@ def main() -> None: if len(ids) > 1: logger.warning(f'Multiple "{token}" tokens found, choosing ID {ids[0]}, use --special-token-by-id if you want another:') - logger.warning(', '.join(ids)) + logger.warning(', '.join(str(i) for i in ids)) for name, id_string in args.special_token_by_id or []: if name not in token_names: From e5956f5bbeb684a5e8cbae318215d82016ff5de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 21 Apr 2024 11:42:12 +0200 Subject: [PATCH 05/17] make script executable --- gguf-py/scripts/gguf-new-metadata.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 gguf-py/scripts/gguf-new-metadata.py diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py old mode 100644 new mode 100755 From ff5d21e608f10a72f8af34591b59e1c9fd24f5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 21 Apr 2024 11:43:43 +0200 Subject: [PATCH 06/17] switch to namedtuple, no need to dataclass --- gguf-py/scripts/gguf-new-metadata.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 3963d13dd48f1..9a0cec64a1aea 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -7,8 +7,7 @@ from pathlib import Path import numpy as np -from typing import Any, Mapping, Sequence -from dataclasses import dataclass +from typing import Any, Mapping, Sequence, NamedTuple # Necessary to load the local gguf package if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): @@ -19,8 +18,7 @@ logger = logging.getLogger("gguf-new-metadata") -@dataclass -class MetadataDetails: +class MetadataDetails(NamedTuple): type: gguf.GGUFValueType value: Any description: str = '' From d39f20359ee3e666c5707e5b99cd2993f6eac7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 4 May 2024 20:19:50 +0200 Subject: [PATCH 07/17] typing++ --- gguf-py/scripts/gguf-new-metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 9a0cec64a1aea..9b7920adf151d 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -7,7 +7,7 @@ from pathlib import Path import numpy as np -from typing import Any, Mapping, Sequence, NamedTuple +from typing import Any, Sequence, NamedTuple # Necessary to load the local gguf package if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): @@ -40,7 +40,7 @@ def get_byteorder(reader: gguf.GGUFReader) -> gguf.GGUFEndian: return host_endian -def decode_field(field: gguf.ReaderField) -> Any: +def decode_field(field: gguf.ReaderField | None) -> Any: if field and field.types: main_type = field.types[0] @@ -74,7 +74,7 @@ def find_token(token_list: Sequence[int], token: str) -> Sequence[int]: return token_ids -def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: +def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: dict[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: for field in reader.fields.values(): # Suppress virtual fields and fields written by GGUFWriter if field.name == gguf.Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'): @@ -115,7 +115,7 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new for tensor in reader.tensors: # Dimensions are written in reverse order, so flip them first - shape = np.flipud(tensor.shape) + shape = np.flipud(tensor.shape).tolist() writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) writer.write_header_to_file() From 158215c828d2f083765baa360a038c12c63c8719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 4 May 2024 20:29:40 +0200 Subject: [PATCH 08/17] add progress bar --- gguf-py/scripts/gguf-new-metadata.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 9b7920adf151d..64068f80c7d37 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -7,6 +7,7 @@ from pathlib import Path import numpy as np +from tqdm import tqdm from typing import Any, Sequence, NamedTuple # Necessary to load the local gguf package @@ -113,17 +114,23 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new writer.add_key(key) writer.add_val(val.value, val.type) + total_bytes = 0 + for tensor in reader.tensors: + total_bytes += tensor.n_bytes # Dimensions are written in reverse order, so flip them first shape = np.flipud(tensor.shape).tolist() writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) + bar = tqdm(desc="Writing", total=total_bytes, unit="byte", unit_scale=True) + writer.write_header_to_file() writer.write_kv_data_to_file() writer.write_ti_data_to_file() for tensor in reader.tensors: writer.write_tensor_data(tensor.data) + bar.update(tensor.n_bytes) writer.close() From ed72533651356205e72c33e1659889cd9c961c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 08:33:54 +0200 Subject: [PATCH 09/17] Add special token modification capability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To be able to fix/amend special tokens in a GGUF let's add two new arguments: * `--special-token ` where `` can be bos, eos, prefix, middle, etc. while `` is the token value, f.ex. `"<｜fim▁begin｜>"` * `--special-token-by-id ` where `` is the ID of the token, f.ex. 32006 So, in order to f.ex. add fill-in-middle tokens to a GGUF you would do the following: ```bash gguf-new-metadata.py input.gguf output.gguf --special-token prefix "<｜fim▁begin｜>" --special-token middle "<｜fim▁end｜>" --special-token suffix "<｜fim▁hole｜>" ``` (yes, fim_end is the `middle` token, because completion is a `prefix`/`suffix`/`middle` sequence (where `middle` is unfilled)) or ```bash gguf-new-metadata.py input.gguf output.gguf --special-token prefix "" --special-token middle "" --special-token suffix "" ``` etc... NB: The tokens have to exist already, trying to add non-existent token name/IDs will be ignored (with a warning), while non-existent values will fail (with an error). --- gguf-py/scripts/gguf-new-metadata.py | 74 ++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index c8e3a83dfbd78..8ef9b15fe0f00 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -8,6 +8,7 @@ import numpy as np from typing import Any, Sequence +from dataclasses import dataclass # Necessary to load the local gguf package if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): @@ -18,6 +19,12 @@ logger = logging.getLogger("gguf-new-metadata") +@dataclass +class MetadataDetails: + type: gguf.GGUFValueType + value: Any + description: str = '' + def get_byteorder(reader: gguf.GGUFReader) -> gguf.GGUFEndian: if np.uint32(1) == np.uint32(1).newbyteorder("<"): # Host is little endian @@ -59,7 +66,16 @@ def get_field_data(reader: gguf.GGUFReader, key: str) -> Any: return decode_field(field) -def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: dict[str, str], remove_metadata: Sequence[str]) -> None: +def find_token(token_list: Sequence[int], token: str) -> Sequence[int]: + token_ids = [index for index, value in enumerate(token_list) if value == token] + + if len(token_ids) == 0: + raise LookupError(f'Unable to find "{token}" in token list!') + + return token_ids + + +def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: for field in reader.fields.values(): # Suppress virtual fields and fields written by GGUFWriter if field.name == gguf.Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'): @@ -75,29 +91,28 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new logger.debug(f'Removing {field.name}') continue - old_val = decode_field(field) + old_val = MetadataDetails(field.types[0], decode_field(field)) val = new_metadata.get(field.name, old_val) if field.name in new_metadata: - logger.debug(f'Modifying {field.name}: "{old_val}" -> "{val}"') + logger.debug(f'Modifying {field.name}: "{old_val.value}" -> "{val.value}" {val.description}') del new_metadata[field.name] - elif val is not None: + elif val.value is not None: logger.debug(f'Copying {field.name}') - if val is not None: + if val.value is not None: writer.add_key(field.name) - writer.add_val(val, field.types[0]) + writer.add_val(val.value, val.type) if gguf.Keys.Tokenizer.CHAT_TEMPLATE in new_metadata: logger.debug('Adding chat template(s)') - writer.add_chat_template(new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE]) + writer.add_chat_template(new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE].value) del new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] - # TODO: Support other types than string? for key, val in new_metadata.items(): - logger.debug(f'Adding {key}: {val}') + logger.debug(f'Adding {key}: "{val.value}" {val.description}') writer.add_key(key) - writer.add_val(val, gguf.GGUFValueType.STRING) + writer.add_val(val.value, val.type) for tensor in reader.tensors: # Dimensions are written in reverse order, so flip them first @@ -115,6 +130,9 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new def main() -> None: + tokenizer_metadata = (getattr(gguf.Keys.Tokenizer, n) for n in gguf.Keys.Tokenizer.__dict__.keys() if not n.startswith('_')) + token_names = dict((n.split('.')[-1][:-len('_token_id')], n) for n in tokenizer_metadata if n.endswith('_token_id')) + parser = argparse.ArgumentParser(description="Make a copy of a GGUF file with new metadata") parser.add_argument("input", type=Path, help="GGUF format model input filename") parser.add_argument("output", type=Path, help="GGUF format model output filename") @@ -123,6 +141,8 @@ def main() -> None: parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)") parser.add_argument("--chat-template-config", type=Path, help="Config file (tokenizer_config.json) containing chat template(s)") parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model") + parser.add_argument("--special-token", action="append", type=str, help="Special token by value", nargs=2, metavar=(' | '.join(token_names.keys()), '""')) + parser.add_argument("--special-token-by-id", action="append", type=str, help="Special token by id", nargs=2, metavar=(' | '.join(token_names.keys()), '0')) parser.add_argument("--force", action="store_true", help="Bypass warnings without confirmation") parser.add_argument("--verbose", action="store_true", help="Increase output verbosity") args = parser.parse_args(None if len(sys.argv) > 2 else ["--help"]) @@ -133,20 +153,20 @@ def main() -> None: remove_metadata = args.remove_metadata or [] if args.general_name: - new_metadata[gguf.Keys.General.NAME] = args.general_name + new_metadata[gguf.Keys.General.NAME] = MetadataDetails(gguf.GGUFValueType.STRING, args.general_name) if args.general_description: - new_metadata[gguf.Keys.General.DESCRIPTION] = args.general_description + new_metadata[gguf.Keys.General.DESCRIPTION] = MetadataDetails(gguf.GGUFValueType.STRING, args.general_description) if args.chat_template: - new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = json.loads(args.chat_template) if args.chat_template.startswith('[') else args.chat_template + new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, json.loads(args.chat_template) if args.chat_template.startswith('[') else args.chat_template) if args.chat_template_config: with open(args.chat_template_config, 'r') as fp: config = json.load(fp) template = config.get('chat_template') if template: - new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = template + new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, template) if remove_metadata: logger.warning('*** Warning *** Warning *** Warning **') @@ -166,6 +186,32 @@ def main() -> None: arch = get_field_data(reader, gguf.Keys.General.ARCHITECTURE) endianess = get_byteorder(reader) + token_list = get_field_data(reader, gguf.Keys.Tokenizer.LIST) or [] + + for name, token in args.special_token or []: + if name not in token_names: + logger.warning(f'Unknown special token "{name}", ignoring...') + else: + ids = find_token(token_list, token) + new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, ids[0], f'= {token}') + + if len(ids) > 1: + logger.warning(f'Multiple "{token}" tokens found, choosing ID {ids[0]}, use --special-token-by-id if you want another:') + logger.warning(', '.join(ids)) + + for name, id_string in args.special_token_by_id or []: + if name not in token_names: + logger.warning(f'Unknown special token "{name}", ignoring...') + elif not id_string.isdecimal(): + logger.warning(f'Token ID "{id_string}" is not a valid ID, ignoring...') + else: + id_int = int(id_string) + + if id_int >= 0 and id_int < len(token_list): + new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, id_int, f'= {token_list[id_int]}') + else: + logger.warning(f'Token ID {id_int} is not within token list, ignoring...') + if os.path.isfile(args.output) and not args.force: logger.warning('*** Warning *** Warning *** Warning **') logger.warning(f'* The "{args.output}" GGUF file already exists, it will be overwritten!') From 27caf19917ada8fb99602356671854a1184005c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 08:45:33 +0200 Subject: [PATCH 10/17] improve help text --- gguf-py/scripts/gguf-new-metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 8ef9b15fe0f00..24e2f54ad4f34 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -136,11 +136,11 @@ def main() -> None: parser = argparse.ArgumentParser(description="Make a copy of a GGUF file with new metadata") parser.add_argument("input", type=Path, help="GGUF format model input filename") parser.add_argument("output", type=Path, help="GGUF format model output filename") - parser.add_argument("--general-name", type=str, help="The models general.name") - parser.add_argument("--general-description", type=str, help="The models general.description") - parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)") - parser.add_argument("--chat-template-config", type=Path, help="Config file (tokenizer_config.json) containing chat template(s)") - parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model") + parser.add_argument("--general-name", type=str, help="The models general.name", metavar='"name"') + parser.add_argument("--general-description", type=str, help="The models general.description", metavar='"Description ..."') + parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)", metavar='"{% ... %} ..."') + parser.add_argument("--chat-template-config", type=Path, help="Config file containing chat template(s)", metavar='tokenizer_config.json') + parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model", metavar='general.url') parser.add_argument("--special-token", action="append", type=str, help="Special token by value", nargs=2, metavar=(' | '.join(token_names.keys()), '""')) parser.add_argument("--special-token-by-id", action="append", type=str, help="Special token by id", nargs=2, metavar=(' | '.join(token_names.keys()), '0')) parser.add_argument("--force", action="store_true", help="Bypass warnings without confirmation") From 8737ca16f37b8e1323d5edb5d228c9307ba0681a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 08:48:39 +0200 Subject: [PATCH 11/17] flake-- --- gguf-py/scripts/gguf-new-metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 24e2f54ad4f34..6b0f22e7630a8 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -25,6 +25,7 @@ class MetadataDetails: value: Any description: str = '' + def get_byteorder(reader: gguf.GGUFReader) -> gguf.GGUFEndian: if np.uint32(1) == np.uint32(1).newbyteorder("<"): # Host is little endian @@ -67,12 +68,12 @@ def get_field_data(reader: gguf.GGUFReader, key: str) -> Any: def find_token(token_list: Sequence[int], token: str) -> Sequence[int]: - token_ids = [index for index, value in enumerate(token_list) if value == token] + token_ids = [index for index, value in enumerate(token_list) if value == token] - if len(token_ids) == 0: - raise LookupError(f'Unable to find "{token}" in token list!') + if len(token_ids) == 0: + raise LookupError(f'Unable to find "{token}" in token list!') - return token_ids + return token_ids def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: From 3e3e7c376afa102190d81d5f7b901da1c304bd8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 20 Apr 2024 09:34:01 +0200 Subject: [PATCH 12/17] fix multiple tokens warning --- gguf-py/scripts/gguf-new-metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 6b0f22e7630a8..de6b2f527955a 100644 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -198,7 +198,7 @@ def main() -> None: if len(ids) > 1: logger.warning(f'Multiple "{token}" tokens found, choosing ID {ids[0]}, use --special-token-by-id if you want another:') - logger.warning(', '.join(ids)) + logger.warning(', '.join(str(i) for i in ids)) for name, id_string in args.special_token_by_id or []: if name not in token_names: From bc92f65e8cab029989611eb507f8e686a0e8e67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 21 Apr 2024 11:42:12 +0200 Subject: [PATCH 13/17] make script executable --- gguf-py/scripts/gguf-new-metadata.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 gguf-py/scripts/gguf-new-metadata.py diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py old mode 100644 new mode 100755 From 87e2d7354f30c341ca93da9e1bc41515b06e8e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 21 Apr 2024 11:43:43 +0200 Subject: [PATCH 14/17] switch to namedtuple, no need to dataclass --- gguf-py/scripts/gguf-new-metadata.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index de6b2f527955a..94ebeabce49ef 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -7,8 +7,7 @@ from pathlib import Path import numpy as np -from typing import Any, Sequence -from dataclasses import dataclass +from typing import Any, Sequence, NamedTuple # Necessary to load the local gguf package if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): @@ -19,8 +18,7 @@ logger = logging.getLogger("gguf-new-metadata") -@dataclass -class MetadataDetails: +class MetadataDetails(NamedTuple): type: gguf.GGUFValueType value: Any description: str = '' From 981bd44e7de202ccde33774ea148c56fb45fa8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 4 May 2024 20:19:50 +0200 Subject: [PATCH 15/17] typing++ --- gguf-py/scripts/gguf-new-metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 94ebeabce49ef..316f9bf1c2fa5 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -74,7 +74,7 @@ def find_token(token_list: Sequence[int], token: str) -> Sequence[int]: return token_ids -def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: +def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: dict[str, MetadataDetails], remove_metadata: Sequence[str]) -> None: for field in reader.fields.values(): # Suppress virtual fields and fields written by GGUFWriter if field.name == gguf.Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'): From 609df3c703dac923b3b00df212f862d0d050a73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 4 May 2024 20:29:40 +0200 Subject: [PATCH 16/17] add progress bar --- gguf-py/scripts/gguf-new-metadata.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 316f9bf1c2fa5..f2a9e8b146289 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -7,6 +7,7 @@ from pathlib import Path import numpy as np +from tqdm import tqdm from typing import Any, Sequence, NamedTuple # Necessary to load the local gguf package @@ -113,17 +114,23 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new writer.add_key(key) writer.add_val(val.value, val.type) + total_bytes = 0 + for tensor in reader.tensors: + total_bytes += tensor.n_bytes # Dimensions are written in reverse order, so flip them first shape = np.flipud(tensor.shape).tolist() writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) + bar = tqdm(desc="Writing", total=total_bytes, unit="byte", unit_scale=True) + writer.write_header_to_file() writer.write_kv_data_to_file() writer.write_ti_data_to_file() for tensor in reader.tensors: writer.write_tensor_data(tensor.data) + bar.update(tensor.n_bytes) writer.close() From efd70f3ce9e79af186d5bd5e276db962d52df29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Thu, 9 May 2024 12:21:11 +0200 Subject: [PATCH 17/17] fail on invalid token id --- gguf-py/scripts/gguf-new-metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index f2a9e8b146289..63d3c5d8fdcf4 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -209,14 +209,14 @@ def main() -> None: if name not in token_names: logger.warning(f'Unknown special token "{name}", ignoring...') elif not id_string.isdecimal(): - logger.warning(f'Token ID "{id_string}" is not a valid ID, ignoring...') + raise LookupError(f'Token ID "{id_string}" is not a valid ID!') else: id_int = int(id_string) if id_int >= 0 and id_int < len(token_list): new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, id_int, f'= {token_list[id_int]}') else: - logger.warning(f'Token ID {id_int} is not within token list, ignoring...') + raise LookupError(f'Token ID {id_int} is not within token list!') if os.path.isfile(args.output) and not args.force: logger.warning('*** Warning *** Warning *** Warning **')