Added codec: hamming

Author: dhondta

README.md

@@ -312,6 +312,7 @@ This category also contains `ascii85`, `adobe`, `[x]btoa`, `zeromq` with the `ba
 - [X] `excess3`: uses Excess-3 (aka Stibitz code) binary encoding to convert characters from their ordinals
 - [X] `fibonacci`: Fibonacci universal code, encoding characters from their ordinals (or lists of integers with `-integers`)
 - [X] `gray`: aka reflected binary code
+- [X] `hamming`: Hamming(7,4) error-correcting code, correcting single-bit errors (and detecting double-bit errors with `hamming84`)
 - [X] `manchester`: XORes each bit of the input with `01`
 - [X] `manchester-inverted`: variant of `manchester` ; XORes each bit of the input with `10`
 - [X] `nrzi`: Non-Return-to-Zero Inverted line code, writing 1 (or 0 with `nrzi-space`) as a transition

docs/pages/enc/binary.md

@@ -186,6 +186,26 @@ Also called *reflected binary code*, it implements the Gray code applied to char
 
 -----
 
+### Hamming
+
+The [Hamming(7,4)](https://en.wikipedia.org/wiki/Hamming(7,4)) code, invented by Richard Hamming in 1950, writes 4 data bits with 3 parity bits, as the codeword `p1 p2 d1 p3 d2 d3 d4`, where `p1 = d1 ^ d2 ^ d4`, `p2 = d1 ^ d3 ^ d4` and `p3 = d2 ^ d3 ^ d4`: the parity checks of a received codeword give the position of a flipped bit, hence a single-bit error is corrected. The extended Hamming(8,4) code (aka SECDED) adds an overall parity bit, so that two flipped bits are detected instead of being wrongly corrected. Each half of a byte is encoded as a codeword in a byte (whose most significant bit is the overall parity bit with Hamming(8,4) and 0 with Hamming(7,4)), hence each input byte gives two output bytes.
+
+**Codec** | **Conversions** | **Aliases** | **Comment**
+:---: | :---: | --- | ---
+`hamming` | data <-> Hamming(7,4) codewords | `hamming74`, `hamming-7-4` | single-bit errors silently corrected (two flipped bits in a codeword are wrongly corrected)
+`hamming84` | data <-> Hamming(8,4) codewords | `hamming-8-4`, `secded` | single-bit errors silently corrected ; double-bit errors (and a trailing byte) only replaced, dropped or left with a non-strict errors handling
+
+```python
+>>> codext.encode(b"Hi", "hamming84")
+b'\xcc\xf0f\x99'
+>>> codext.decode(b"\xc8\xf0f\x99", "hamming84")  # with a flipped bit in the first codeword
+b'Hi'
+>>> codext.decode(b"\x03\xc3", "hamming84", "replace")  # with two flipped bits in the first codeword
+b'?'
+```
+
+-----
+
 ### Manchester
 
 This codec XORes each group of 4 bits of the input text with a 1-byte clock signal, e.g. `0x55` giving in binary `01010101`.

src/codext/binary/__init__.py

@@ -6,6 +6,7 @@
 from .excess3 import *
 from .fibonacci import *
 from .gray import *
+from .hamming import *
 from .manchester import *
 from .nrzi import *
 from .rotate import *

src/codext/binary/hamming.py

@@ -0,0 +1,87 @@
+# -*- coding: UTF-8 -*-
+"""Hamming Codec - Hamming(7,4) error-correcting content encoding.
+
+The Hamming(7,4) code, invented by Richard Hamming in 1950, writes 4 data bits (d1 to d4) with 3 parity bits (p1, p2
+ and p3), as the codeword p1 p2 d1 p3 d2 d3 d4, where p1 = d1 ^ d2 ^ d4, p2 = d1 ^ d3 ^ d4 and p3 = d2 ^ d3 ^ d4: the
+ parity checks of a received codeword (its syndrome) give the position of a flipped bit, hence a single-bit error is
+ corrected. The extended Hamming(8,4) code (aka SECDED, for single error correction and double error detection) adds an
+ overall parity bit, so that two flipped bits are detected (instead of being wrongly corrected).
+
+Each half of a byte (from the most significant one) is encoded as a codeword in a byte (whose most significant bit is
+ the overall parity bit with Hamming(8,4), and 0 with Hamming(7,4)), hence each input byte gives two output bytes.
+ Decoding silently corrects the single-bit errors ; with Hamming(8,4), the double-bit errors are reported (or, with a
+ non-strict errors handling, replaced, dropped or left as is), as well as a trailing byte.
+
+This codec:
+- en/decodes strings from str to str
+- en/decodes strings from bytes to bytes
+- decodes file content to str (read)
+- encodes file content from str to bytes (write)
+
+Reference: https://en.wikipedia.org/wiki/Hamming(7,4)
+"""
+from ..__common__ import *
+
+
+__examples1__ = {
+    'enc(hamming|hamming74|hamming-7-4|hamming_7_4)': {'\xb3': "3C", 'A': "Li", '\x00\xff': "\x00\x00\x7f\x7f"},
+    'dec(hamming)': {'3C': "\xb3", '\x13C': "\xb3", '3B': "\xb3", '\xb3C': "\xb3", '3': None},
+    'enc-dec(hamming)': ["", "@random{512}"],
+}
+__examples2__ = {
+    'enc(hamming84|hamming-8-4|secded)': {'\xb3': "3\xc3", 'A': "\xcci", '\x00\xff': "\x00\x00\xff\xff"},
+    'dec(hamming84)': {'3\xc3': "\xb3", '\x13\xc3': "\xb3", '\xb3\xc3': "\xb3", '\x03\xc3': None, '3': None},
+    'enc-dec(hamming84)': ["", "@random{512}"],
+}
+
+
+def _codeword(nibble, extended):
+    d1, d2, d3, d4 = (nibble >> 3) & 1, (nibble >> 2) & 1, (nibble >> 1) & 1, nibble & 1
+    c = (d1 ^ d2 ^ d4) << 6 | (d1 ^ d3 ^ d4) << 5 | d1 << 4 | (d2 ^ d3 ^ d4) << 3 | d2 << 2 | d3 << 1 | d4
+    # the overall parity bit (Hamming(8,4)) is the most significant bit of the byte
+    return c | (bin(c).count("1") & 1) << 7 if extended else c
+
+
+def _decoding(extended):
+    """ Table of the decoded nibbles of the bytes, None for the uncorrectable errors. """
+    table = []
+    for byte in range(256):
+        c = byte & 0x7f
+        bits = [c >> (6 - i) & 1 for i in range(7)]
+        # the syndrome is the position (from 1) of the flipped bit, 0 if none
+        s = (bits[0] ^ bits[2] ^ bits[4] ^ bits[6]) | (bits[1] ^ bits[2] ^ bits[5] ^ bits[6]) << 1 | \
+            (bits[3] ^ bits[4] ^ bits[5] ^ bits[6]) << 2
+        # with the overall parity, an even number of flipped bits with a non-null syndrome is a double-bit error
+        if extended and s and not bin(byte).count("1") & 1:
+            table.append(None)
+            continue
+        if s:
+            bits[s - 1] ^= 1
+        table.append(bits[2] << 3 | bits[4] << 2 | bits[5] << 1 | bits[6])
+    return table
+
+
+def hamming_encode(extended):
+    codewords = [_codeword(n, extended) for n in range(16)]
+    def encode(text, errors="strict"):
+        return bytes(codewords[x >> i & 15] for x in b(ensure_str(text)) for i in (4, 0)).decode("latin-1"), len(text)
+    return encode
+
+
+def hamming_decode(extended):
+    ename, table = "hamming84" if extended else "hamming", _decoding(extended)
+    def decode(text, errors="strict"):
+        _h, data, r = handle_error(ename, errors, decode=True, kind="codewords"), b(ensure_str(text)), ""
+        # a trailing byte is reported first, then (with a non-strict errors handling) replaced, dropped or left as is
+        #  at the end
+        trailing = _h(data[-1:].decode("latin-1"), len(data) - 1, r) if len(data) % 2 else ""
+        for i in range(0, len(data) - 1, 2):
+            hi, lo = table[data[i]], table[data[i+1]]
+            r += _h(data[i:i+2].decode("latin-1"), i, r) if hi is None or lo is None else chr(hi << 4 | lo)
+        return (r + trailing).encode("latin-1"), len(text)
+    return decode
+
+
+add("hamming", hamming_encode(False), hamming_decode(False), r"^hamming(?:[-_]?7[-_]?4)?$", examples=__examples1__)
+add("hamming84", hamming_encode(True), hamming_decode(True), r"^(?:hamming[-_]?8[-_]?4|secded)$",
+    examples=__examples2__)

src/codext/binary/manchester.py

@@ -93,7 +93,8 @@ def _code(text, errors="strict"):
                     _h(f"{h1}{h2}", 8 * j + 14 - 2 * i)
                 y, level = y << 1 | (transition if h1 != level else 1 - transition), h2
             r.append(y)
-        return bytes(r).decode("latin-1"), len(text)
+        # the decoded data is given as bytes (then as text if it was text)
+        return bytes(r), len(text)
     return _code
 
 

src/codext/binary/nrzi.py

@@ -43,7 +43,8 @@ def _code(text, errors="strict"):
                     level ^= bit == transition
                     y = y << 1 | level
             r.append(y)
-        return bytes(r).decode("latin-1"), len(text)
+        # the encoded levels are given as latin-1 text, the decoded data as bytes (then as text if it was text)
+        return (bytes(r) if decode else bytes(r).decode("latin-1")), len(text)
     return _code
 
 

tests/test_manual.py

@@ -140,11 +140,39 @@ def recording(keys, rate=8000, width=2, channels=1, tone=.1, pause=.1, drift=0.,
     def test_codec_morse(self):
         self.assertRaises(LookupError, codecs.encode, "test", "morse-AAB")
 
+    def test_codec_hamming(self):
+        import random
+        from itertools import combinations
+        from codext.binary.hamming import _codeword, _decoding
+        # example of Wikipedia, minimum distances (3 and 4), correction of all the single-bit errors and, with
+        #  Hamming(8,4), detection of all the double-bit errors (that Hamming(7,4) wrongly corrects)
+        self.assertEqual(_codeword(0b1011, False), 0b0110011)
+        for extended, n, distance in [(False, 7, 3), (True, 8, 4)]:
+            codewords, table = [_codeword(d, extended) for d in range(16)], _decoding(extended)
+            self.assertEqual(min(bin(a ^ c).count("1") for a, c in combinations(codewords, 2)), distance)
+            for d, c in enumerate(codewords):
+                self.assertTrue(all(table[c ^ 1 << i] == d for i in range(8)))
+                for i, j in combinations(range(n), 2):
+                    r = table[c ^ 1 << i ^ 1 << j]
+                    self.assertTrue(r is None if extended else r != d)
+        # a message with a flipped bit in each codeword is fully corrected
+        text, rnd = "The quick brown fox jumps over the lazy dog".encode() * 10, random.Random(1)
+        for e in ["hamming", "hamming84"]:
+            noisy = bytes(c ^ 1 << rnd.randrange(8 if e == "hamming84" else 7) for c in codecs.encode(text, e))
+            self.assertEqual(codecs.decode(noisy, e), text)
+        # with a non-strict errors handling, double-bit errors are replaced, dropped or left as is, as a trailing byte
+        for e, r in [("replace", b"A??"), ("ignore", b"A"), ("leave", b"A\x03\xc3\x01")]:
+            self.assertEqual(codecs.decode(b"\xcci\x03\xc3\x01", "hamming84", e), r)
+        self.assertRaises(ValueError, codecs.decode, b"\xcci\x03\xc3", "hamming84")
+        self.assertRaises(ValueError, codecs.decode, b"\xcci\x01", "hamming84", "BAD_ERRORS")
+    
     def test_codec_nrzi_differential_manchester(self):
         # these line codes do not depend on the polarity of the signal: the inverted signal gives the same bits, except
         #  the first one (depending on the initial level)
         data = bytes(range(256))
         for e in ["nrzi", "nrzi-space", "differential-manchester", "differential-manchester-inverted"]:
+            # text with characters outside latin-1 is given back
+            self.assertEqual(codecs.decode(codecs.encode("caf\u00e9 \u20ac", e), e), "caf\u00e9 \u20ac")
             inverted = bytes(x ^ 0xff for x in codecs.encode(data, e))
             r = codecs.decode(inverted, e)
             self.assertEqual(r[0] & 0x7f, data[0] & 0x7f)