Added codec: base128

Author: dhondta

README.md

@@ -283,6 +283,7 @@ CACTCGGTCGGCCATATGTTCGGCCATATGTTCGTCTGTTCACTCGCCCATACACT
 - [X] `base85`: all variants of Base85 ([Ascii85](https://fr.wikipedia.org/wiki/Ascii85), [z85](https://rfc.zeromq.org/spec/32), [Adobe](https://dencode.com/string/ascii85), [(x)btoa](https://dencode.com/string/ascii85), [RFC1924](https://datatracker.ietf.org/doc/html/rfc1924), [XML](https://datatracker.ietf.org/doc/html/draft-kwiatkowski-base85-for-xml-00))
 - [X] `base91`: [Base91](http://base91.sourceforge.net) custom conversion
 - [X] `base100` (or *emoji*): [Base100](https://github.com/AdamNiederer/base100) custom conversion
+- [X] `base128-latin1`: [Base128](https://github.com/rpuntaie/base128) with printable characters (digits, letters and Latin-1)
 - [X] `base122`: [Base100](http://blog.kevinalbs.com/base122) custom conversion
 - [X] `base1024` (or *ecoji*): [Ecoji](https://github.com/keith-turner/ecoji) custom conversion (10 bits per emoji, versions 1 and 2)
 - [X] `base2048`: [Base2048](https://github.com/qntm/base2048) custom conversion (11 bits per character, optimised for Twitter)

docs/pages/enc/base.md

@@ -173,6 +173,7 @@ This encoding implements various different versions of Base85.
 `base100` | text <-> Base100 encoded text | `base[-_]?100|emoji` | Python 3 only
 `base122` | text <-> Base122 encoded text | `base[-_]?122` | Python 3 only
 `base128` | text <-> Base128 encoded text | `base[-_]?128` | Relies on the ASCII charset
+`base128-latin1` | data <-> Base128 encoded text | `base128[-_]latin[-_]?1` | [Base128](https://github.com/rpuntaie/base128) with printable characters (digits, letters and Latin-1) ; chunks of 7 bytes, 7 bits per character, a last character giving the length of the last chunk
 `leb128` | integers <-> LEB128 (varint) bytes | `uleb128`, `varint` | unsigned integers, separated by whitespaces or commas
 `sleb128` | integers <-> signed LEB128 bytes | `leb128-signed` | signed integers (two's complement)
 `base1024` | data <-> Ecoji encoded text | `base[-_]?1024`, `ecoji`, `ecoji-v1`, `ecoji-v2` | 10 bits per emoji ; encodes with Ecoji version 2 by default
@@ -196,6 +197,13 @@ This encoding implements various different versions of Base85.
 'this is a test'
 ```
 
+```python
+>>> codecs.encode("Hello World!", "base128-latin1")
+'aPjÃây·ÕtÚÊÃX15'
+>>> codecs.decode("aPjÃây·ÕtÚÊÃX15", "base128-latin1")
+'Hello World!'
+```
+
 Base2048, Base32768 and Base65536 were designed by [qntm](https://github.com/qntm/base65536). The data is written with 11, 15 or 16 bits per character, using repertoires of Unicode characters that are safe to transmit (no whitespace, no punctuation, no control or combining characters). The final bits are padded with 1s and may be written with a character of a smaller final repertoire, which can only end the encoded text. When decoding, whitespaces are ignored.
 
 ```python

pyproject.toml

@@ -107,6 +107,7 @@ base85-zeromq = "codext.base.base85:main85zeromq"
 base91 = "codext.base.base91:main91"
 base100 = "codext.base.base100:main100"
 base122 = "codext.base.base122:main122"
+base128-latin1 = "codext.base.base128:main128latin1"
 base1024 = "codext.base.base1024:main1024"
 base2048 = "codext.base.base65536:main2048"
 base32768 = "codext.base.base65536:main32768"

src/codext/base/__init__.py

@@ -9,6 +9,7 @@
 from .base91 import *
 from .base100 import *
 from .base122 import *
+from .base128 import *
 from .base1024 import *
 from .bech32 import *
 from .base65536 import *

src/codext/base/base128.py

@@ -0,0 +1,80 @@
+# -*- coding: UTF-8 -*-
+"""Base128-Latin1 Codec - base128-latin1 content encoding.
+
+This Base128 encoding, unlike the "base128" codec (relying on the ASCII charset, hence holding control characters), uses
+ a set of 128 printable characters (digits, letters and Latin-1 characters). The data is taken by
+ chunks of 7 bytes, each chunk being split into 8 groups of 7 bits written with 1 character each ; the bits of a last
+ incomplete chunk are split the same way, those of its last group being right-aligned. A last character gives the length
+ of the last chunk (0 when it is complete). Whitespace is ignored when decoding.
+
+This codec:
+- en/decodes strings from str to str
+- en/decodes strings from bytes to bytes
+- decodes file content to str (read)
+- encodes file content from str to bytes (write)
+
+Reference: https://github.com/rpuntaie/base128
+"""
+from ._base import main
+from ..__common__ import *
+
+
+__examples__ = {
+    'enc(base128-latin1|base128_latin1|base-128-latin-1)': {
+        '': "0", '\x00' * 7: "000000000", '\xff' * 7: "ÿ" * 8 + "0", '\x01': "011", '\xff': "ÿ11",
+        'Hello World!': "aPjÃây·ÕtÚÊÃX15",
+    },
+    'dec(base128-latin1)': {
+        'aPjÃây·ÕtÚÊÃX15': "Hello World!", '011': "\x01", '0': "", '': None, '7': None, '1': None, '01': None,
+        '0111': None, '021': None, '01!1': None, '0000000000': None, '0000000001': None,
+    },
+    'enc-dec(base128-latin1)': ["\xff", "\xff\xfe\xfd\xfc\xfb\xfa", "\xff\xfe\xfd\xfc\xfb\xfa\xf9",
+                                "@random{512,1024}"],
+}
+__guess__ = ["base128-latin1"]
+
+
+B128 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzµ¶" \
+       "·¼½¾ÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÑÒÓÔÕÖ×ØÙÚÛÜÝÞßáâãäåæçèéêëìíîïñòóôõö÷øùúûüýþÿ"
+
+
+def base128_latin1_encode(text, errors="strict"):
+    t, r = b(text), []
+    for i in range(0, len(t), 7):
+        chunk = t[i:i+7]
+        n, v = 8 * len(chunk), int.from_bytes(chunk, "big")
+        # groups of 7 bits from the most significant ones, the last group of an incomplete chunk being right-aligned
+        for k in range(n - 7, -7, -7):
+            r.append(B128[v >> k & 0x7f if k >= 0 else v & (1 << (k + 7)) - 1])
+    r.append(B128[len(t) % 7])
+    return (s := "".join(r)), len(s)
+
+
+def base128_latin1_decode(text, errors="strict"):
+    _h, text, r = handle_error("base128-latin1", errors, decode=True), "".join(ensure_str(text).split()), b""
+    # a wrong character cannot be replaced in a chunk, hence it is dropped
+    t = "".join(c if c in B128 else _h(c, i) and "" for i, c in enumerate(text))
+    if len(t) == 0 or (m := B128.index(t[-1])) > 6:
+        return b(_h(t[-1:], len(t) - 1)), len(text)
+    # the last character gives the length of the last chunk of data (m bytes, encoded with ceil(8m/7) characters)
+    t = t[:-1]
+    if m and len(t) == 0:
+        return b(_h(text, 0)), len(text)
+    for i in range(0, len(t), 8):
+        chunk, n = t[i:i+8], m if m and i + 8 >= len(t) else 7
+        groups = [B128.index(c) for c in chunk]
+        # the last group of an incomplete chunk only holds the remaining bits, right-aligned
+        rbits = 8 * n - 7 * (len(chunk) - 1)
+        if len(chunk) != (8 if n == 7 else -(-8 * m // 7)) or groups[-1] >> rbits:
+            r += b(_h(chunk, i, r))
+            continue
+        v = 0
+        for g in groups[:-1]:
+            v = v << 7 | g
+        r += (v << rbits | groups[-1]).to_bytes(n, "big")
+    return r, len(r)
+
+
+add("base128-latin1", base128_latin1_encode, base128_latin1_decode, r"^base[-_]?128[-_]latin[-_]?1$",
+    expansion_factor=8 / 7)
+main128latin1 = main(128, "<https://github.com/rpuntaie/base128>", "latin1", inv=False, swap=False)

src/codext/stegano/sms.py

@@ -24,8 +24,8 @@
 ENCMAP = {
     ' ': "0", 'a': "2", 'b': "22", 'c': "222", 'd': "3", 'e': "33", 'f': "333", 'g': "4", 'h': "44", 'i': "444",
     'j': "5", 'k': "55", 'l': "555", 'm': "6", 'n': "66", 'o': "666", 'p': "7", 'q': "77", 'r': "777", 's': "7777",
-    't': "8", 'u': "88", 'v': "888", 'w': "9", 'x': "99", 'y': "999", 'z': "9999", '*': "*", '#': "#", ' ': " ",
-    '_': "1", ':': "11", '@': "111",
+    't': "8", 'u': "88", 'v': "888", 'w': "9", 'x': "99", 'y': "999", 'z': "9999", '*': "*", '#': "#", '_': "1",
+    ':': "11", '@': "111",
 }
 
 

tests/test_base.py

@@ -252,6 +252,19 @@ def test_codec_base69(self):
         # non-Base69 characters are dropped
         self.assertEqual(codecs.decode(enc[:5] + "!" + enc[5:], "base69", "ignore"), "ABCDEFGHIJKLMN")
 
+    def test_codec_base128_latin1(self):
+        # a bad chunk is handled as a whole, other chunks being kept ; non-Base128 characters are dropped
+        enc = codecs.encode("ABCDEFGHIJ", "base128-latin1")
+        self.assertEqual(codecs.decode(enc[:11] + "ÿ" + enc[12:], "base128-latin1", "ignore"), "ABCDEFG")
+        self.assertEqual(codecs.decode(enc[:6] + "\n" + enc[6:], "base128-latin1"), "ABCDEFGHIJ")
+        self.assertEqual(codecs.decode(enc[:3] + "!" + enc[3:], "base128-latin1", "ignore"), "ABCDEFGHIJ")
+        # the last character must be a length from 0 to 6, and give the length of the last chunk
+        self.assertEqual(codecs.decode(enc[:-1] + "7", "base128-latin1", "ignore"), "")
+        self.assertEqual(codecs.decode("1", "base128-latin1", "ignore"), "")
+        self.assertRaises(ValueError, codecs.decode, enc[:-1] + "2", "base128-latin1")
+        # the existing base128 codec (ASCII charset) is not affected
+        self.assertEqual(codecs.encode("Hello World!", "base128"), "\t\x06+1Xo\x10\x15mw\x131H!")
+    
     def test_codec_base45(self):
         # RFC 9285 test vectors
         for s, b45 in [("AB", "BB8"), ("Hello!!", "%69 VD92EX0"), ("base-45", "UJCLQE7W581")]: