resolve comments

Author: ds5t5

convert-refact-hf-to-gguf.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# HF falcon--> gguf conversion
+# HF refact--> gguf conversion
 
 from __future__ import annotations
 
@@ -15,8 +15,8 @@
 import torch
 from transformers import AutoTokenizer  # type: ignore[import]
 
-if 'NO_LOCAL_GGUF' not in os.environ:
-    sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
+if "NO_LOCAL_GGUF" not in os.environ:
+    sys.path.insert(1, str(Path(__file__).parent / "gguf-py" / "gguf"))
 import gguf
 
 
@@ -31,13 +31,17 @@ def bytes_to_unicode():
     To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
     And avoids mapping to whitespace/control characters the bpe code barfs on.
     """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    bs = (
+        list(range(ord("!"), ord("~") + 1))
+        + list(range(ord("¡"), ord("¬") + 1))
+        + list(range(ord("®"), ord("ÿ") + 1))
+    )
     cs = bs[:]
     n = 0
     for b in range(2**8):
         if b not in bs:
             bs.append(b)
-            cs.append(2**8+n)
+            cs.append(2**8 + n)
             n += 1
     return dict(zip(bs, (chr(n) for n in cs)))
 
@@ -54,32 +58,41 @@ def count_model_parts(dir_model: Path) -> int:
 
 
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Convert a Refact model to a GGML compatible file")
+    parser = argparse.ArgumentParser(
+        description="Convert a Refact model to a GGML compatible file"
+    )
     parser.add_argument(
-        "--vocab-only", action="store_true",
+        "--vocab-only",
+        action="store_true",
         help="extract only the vocab",
     )
     parser.add_argument(
-        "--outfile", type=Path,
+        "--outfile",
+        type=Path,
         help="path to write to; default: based on input",
     )
     parser.add_argument(
-        "model", type=Path,
+        "model",
+        type=Path,
         help="directory containing model file, or model file itself (*.bin)",
     )
     parser.add_argument(
-        "ftype", type=int, choices=[0, 1], default=1, nargs='?',
+        "ftype",
+        type=int,
+        choices=[0, 1],
+        default=1,
+        nargs="?",
         help="output format - use 0 for float32, 1 for float16",
     )
     return parser.parse_args()
 
+
 args = parse_args()
 
 dir_model = args.model
 ftype = args.ftype
 if not dir_model.is_dir():
-    
-    print(f'Error: {args.model} is not a directory', file = sys.stderr)
+    print(f"Error: {args.model} is not a directory", file=sys.stderr)
     sys.exit(1)
 
 # possible tensor data types
@@ -93,9 +106,9 @@ def parse_args() -> argparse.Namespace:
     fname_out = args.outfile
 else:
     # output in the same directory as the model by default
-    fname_out = dir_model / f'ggml-model-{ftype_str[ftype]}.gguf'
+    fname_out = dir_model / f"ggml-model-{ftype_str[ftype]}.gguf"
 
-print("gguf: loading model "+dir_model.name)
+print("gguf: loading model " + dir_model.name)
 
 with open(dir_model / "config.json", "r", encoding="utf-8") as f:
     hparams = json.load(f)
@@ -108,7 +121,7 @@ def parse_args() -> argparse.Namespace:
 # get number of model parts
 num_parts = count_model_parts(dir_model)
 
-ARCH=gguf.MODEL_ARCH.REFACT
+ARCH = gguf.MODEL_ARCH.REFACT
 gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
 
 print("gguf: get model metadata")
@@ -142,9 +155,9 @@ def parse_args() -> argparse.Namespace:
 scores: list[float] = []
 toktypes: list[int] = []
 
-tokenizer_json_file = dir_model / 'tokenizer.json'
+tokenizer_json_file = dir_model / "tokenizer.json"
 if not tokenizer_json_file.is_file():
-    print(f'Error: Missing {tokenizer_json_file}', file = sys.stderr)
+    print(f"Error: Missing {tokenizer_json_file}", file=sys.stderr)
     sys.exit(1)
 
 # gpt2 tokenizer
@@ -157,7 +170,11 @@ def parse_args() -> argparse.Namespace:
 
 # The number of tokens in tokenizer.json can differ from the expected vocab size.
 # This causes downstream issues with mismatched tensor sizes when running the inference
-vocab_size = hparams["vocab_size"] if "vocab_size" in hparams else len(tokenizer_json["model"]["vocab"])
+vocab_size = (
+    hparams["vocab_size"]
+    if "vocab_size" in hparams
+    else len(tokenizer_json["model"]["vocab"])
+)
 
 tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
 
@@ -176,29 +193,29 @@ def parse_args() -> argparse.Namespace:
                 if ord(c) < 256:  # single byte character
                     text.append(byte_decoder[ord(c)])
                 else:  # multibyte special token character
-                    text.extend(c.encode('utf-8'))
+                    text.extend(c.encode("utf-8"))
     else:
         print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
         pad_token = f"[PAD{i}]".encode("utf8")
         text = bytearray(pad_token)
 
     tokens.append(text)
-    scores.append(0.0)                      # dymmy
+    scores.append(0.0)  # dymmy
     toktypes.append(gguf.TokenType.NORMAL)  # dummy
 
 gguf_writer.add_token_list(tokens)
 gguf_writer.add_token_scores(scores)
 gguf_writer.add_token_types(toktypes)
 
-special_vocab = gguf.SpecialVocab(dir_model, load_merges = True)
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
 special_vocab.add_to_gguf(gguf_writer)
 
 # TENSORS
 
-tensor_map = gguf.get_tensor_name_map(ARCH,block_count)
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
 
 # params for qkv transform
-n_head    = hparams["n_head"]
+n_head = hparams["n_head"]
 n_head_kv = 1
 
 head_dim = hparams["n_embd"] // n_head
@@ -230,7 +247,7 @@ def parse_args() -> argparse.Namespace:
         data = data.squeeze().numpy()
 
         # map tensor names
-        new_name = tensor_map.get_name(name, try_suffixes = (".weight", ))
+        new_name = tensor_map.get_name(name, try_suffixes=(".weight",))
         if new_name is None:
             print("Can not map tensor '" + name + "'")
             sys.exit()
@@ -247,10 +264,23 @@ def parse_args() -> argparse.Namespace:
             data = data.astype(np.float32)
 
         # if f16 desired, convert any float32 2-dim weight tensors to float16
-        if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
+        if (
+            ftype == 1
+            and data_dtype == np.float32
+            and name.endswith(".weight")
+            and n_dims == 2
+        ):
             data = data.astype(np.float16)
 
-        print(new_name + ", n_dims = " + str(n_dims) + ", " + str(old_dtype) + " --> " + str(data.dtype))
+        print(
+            new_name
+            + ", n_dims = "
+            + str(n_dims)
+            + ", "
+            + str(old_dtype)
+            + " --> "
+            + str(data.dtype)
+        )
 
         gguf_writer.add_tensor(new_name, data)
 

llama.cpp

@@ -4129,7 +4129,7 @@ static bool llama_eval_internal(
     // If all tensors can be run on the GPU then using more than 1 thread is detrimental.
     const bool full_offload_supported = model.arch == LLM_ARCH_LLAMA ||
         model.arch == LLM_ARCH_BAICHUAN ||
-        model.arch == LLM_ARCH_FALCON || 
+        model.arch == LLM_ARCH_FALCON ||
         model.arch == LLM_ARCH_REFACT;
     const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 3;
     if (ggml_cpu_has_cublas() && full_offload_supported && fully_offloaded) {