v4.2.6: Self-describing format, byte pipeline, and a performance pass

Every key file and ciphertext now carries a header that states how it was
produced: magic, two fixed markers, and a complete JSON configuration record.
A reader takes its parameters from the artefact rather than from the caller.

This is an incompatible format change. Headerless input is rejected with E017;
pre-4.2.6 material is read through a compatibility API that wraps it in a
minimal header and then uses the ordinary path, so the strict path stays strict.

Added
- Self-describing header on all artefacts, with a normative specification in
  docs/FORMAT.md
- E017 MissingMagicHeader and E018 MalformedHeader
- Byte pipeline for turbo and raw: the XOR layers exchange raw bytes rather
  than hexadecimal text, cutting expansion from 5.33x to 2.00x and raising
  throughput about 4x. The old behaviour is available as mid="hex".
- Selectable output encoding for turbo and raw: hex (2.00x) or base64 (1.33x)
- Legacy compatibility API: wrap_legacy, decrypt_legacy, decrypt_legacy_auto,
  load_legacy_keys
- Frozen reference vectors covering every mode, encoding and intermediate
- docs/benchmark.md with measured throughput and the profiling behind it

Changed
- Default mode is fast: byte-for-byte compatible with the original algorithm,
  roughly 30x quicker
- Substitution, case flipping and the printable fold use bytes.translate over
  precomputed tables
- XOR layers use one big-integer operation instead of a per-byte loop
- Keystream derivation caches length-independent base material; expansion uses
  cached translation tables and list accumulation rather than str +=
- _get_dynamic_key repeats by multiplication. The loop version was quadratic:
  covering a 5.6 MB intermediate took 866 ms, now 3 ms, output unchanged.
- _derive_final_key decodes only the prefix the output consumes
- The decrypt path no longer copies or scans the whole artefact: a redundant
  strip(), a full-string int(payload, 16), and three whole-string slices in
  strip_header were removed. Decryption at 512 KiB went from 34 to 87 MB/s.
- Documentation rewritten for 4.2.6; all eleven markdown files are English

Fixed
1. Key derivation is no longer quadratic (_get_dynamic_key and
   _expand_keystream_slow shared the str += pattern)
2. Two hex-intermediate pipelines were asymmetric, so a turbo or raw artefact
   written with mid="hex" could not be read back
3. change_key_password and export_key_text wrote artefacts without a header
4. The constructor accepted encoding="base64" while the header records
   out="base"; both spellings are now accepted
5. A payload that is not valid hexadecimal reports E010 (MalformedCiphertext)
   rather than E011 (DecryptionFailed)
6. The TUI encoding menu referenced a missing translation

Security
- Format break from pre-4.2.6 artefacts, documented rather than accidental
- fast is not a weaker mode: the frozen vectors assert identical output
- The header adds no secrecy and is not authenticated; an application that must
  detect header tampering should MAC the whole artefact

Performance
- raw with the default bt intermediate: ~60 MB/s encryption, ~70 MB/s
  decryption, ciphertext 2.00x the payload
- Throughput holds steady from 64 KiB to 1 GiB
- The original implementation managed 0.08 MB/s
This commit is contained in:
dvs
2026-09-27 22:06:34 +08:00
parent 24a36d6fb4
commit dd49043171
25 changed files with 5348 additions and 852 deletions
+92 -17
View File
@@ -73,8 +73,9 @@ class TestRoundTrip:
ct = loaded.encrypt(text, USER_PWD)
assert loaded.decrypt(ct, USER_PWD) == text
def test_ciphertext_is_hex(self, loaded):
ct = loaded.encrypt("data", USER_PWD)
def test_ciphertext_payload_is_hex(self, loaded):
from chaoscrypt._header import strip_header
ct = strip_header(loaded.encrypt("data", USER_PWD))
int(ct, 16)
def test_empty_plaintext_rejected(self, loaded):
@@ -98,21 +99,37 @@ class TestRoundTrip:
b = loaded.encrypt("same", "pwd-B")
assert a != b
def test_non_hex_ciphertext_rejected(self, loaded):
def test_every_ciphertext_carries_a_header(self, loaded):
from chaoscrypt._header import MAGIC, read_header
ct = loaded.encrypt("data", USER_PWD)
assert ct.startswith(MAGIC.decode())
cfg = read_header(ct)
assert cfg["kind"] == "ciphertext"
def test_headerless_ciphertext_rejected(self, loaded):
"""Input without the magic header is not a CES artefact."""
with pytest.raises(ChaosCryptError) as ei:
loaded.decrypt("zzzz-not-hex", USER_PWD)
assert ei.value.code == "E010"
assert ei.value.code == "E017"
def test_odd_length_ciphertext_rejected(self, loaded):
def test_headerless_odd_length_rejected(self, loaded):
with pytest.raises(ChaosCryptError) as ei:
loaded.decrypt("abc", USER_PWD)
assert ei.value.code == "E010"
assert ei.value.code == "E017"
def test_empty_ciphertext_rejected(self, loaded):
with pytest.raises(ChaosCryptError) as ei:
loaded.decrypt("", USER_PWD)
assert ei.value.code == "E009"
def test_malformed_payload_rejected(self, loaded):
"""A valid header with a broken payload fails on the payload."""
from chaoscrypt._header import build_header
bad = build_header(loaded.core.header_config("ciphertext")) + "zzzz"
with pytest.raises(ChaosCryptError) as ei:
loaded.decrypt(bad, USER_PWD)
assert ei.value.code == "E010"
# ---------------------------------------------------------------------------
# Binary
@@ -148,6 +165,33 @@ class TestKeyManagement:
n.load(path, KEY_PWD)
assert n.loaded
def test_key_file_carries_a_header(self, tmp_path):
from chaoscrypt._header import MAGIC, read_header
path = str(tmp_path / "hdr.key")
ChaosCrypt().create_key(KEY_PWD, path)
text = open(path, encoding="utf-8").read()
assert text.startswith(MAGIC.decode())
cfg = read_header(text)
assert cfg["kind"] == "key"
assert cfg["alg"] == "chaoscrypt"
def test_headerless_key_file_rejected(self, tmp_path):
path = tmp_path / "bare.key"
path.write_text("deadbeef" * 200, encoding="utf-8")
with pytest.raises(ChaosCryptError) as ei:
ChaosCrypt().load(str(path), KEY_PWD)
assert ei.value.code == "E017"
def test_key_file_rejected_as_ciphertext(self, tmp_path):
path = str(tmp_path / "k2.key")
ChaosCrypt().create_key(KEY_PWD, path)
text = open(path, encoding="utf-8").read()
cc = ChaosCrypt()
cc.load(path, KEY_PWD)
with pytest.raises(ChaosCryptError) as ei:
cc.decrypt(text, USER_PWD)
assert ei.value.code == "E018"
def test_keygen_refuses_overwrite(self, tmp_path):
path = str(tmp_path / "k.key")
ChaosCrypt().create_key(KEY_PWD, path)
@@ -179,14 +223,14 @@ class TestKeyManagement:
path.write_text("this is not a chaoscrypt key file!!!", encoding="utf-8")
with pytest.raises(ChaosCryptError) as ei:
ChaosCrypt().load(str(path), KEY_PWD)
assert ei.value.code == "E002"
assert ei.value.code == "E017"
def test_load_empty_file(self, tmp_path):
path = tmp_path / "empty.key"
path.write_text("", encoding="utf-8")
with pytest.raises(ChaosCryptError) as ei:
ChaosCrypt().load(str(path), KEY_PWD)
assert ei.value.code == "E002"
assert ei.value.code in ("E002", "E017")
def test_encrypt_without_key_raises(self):
cc = ChaosCrypt()
@@ -223,10 +267,13 @@ class TestKeyManagement:
assert len(info["flip_pattern"]) == 10
def test_export_and_import_key_text(self):
from chaoscrypt._header import MAGIC, read_header
cc = ChaosCrypt()
text = cc.export_key_text(KEY_PWD)
assert text and isinstance(text, str)
assert cc.loaded, "export_key_text should leave the instance usable"
assert text.startswith(MAGIC.decode())
assert read_header(text)["kind"] == "key"
n = ChaosCrypt()
n.load_from_text(text, KEY_PWD)
@@ -283,6 +330,13 @@ class TestFileAPI:
back = loaded.decrypt_file(cc_file, USER_PWD, dst)
assert open(back, "rb").read() == payload
def test_encrypted_file_carries_a_header(self, loaded, tmp_path):
from chaoscrypt._header import MAGIC
src = tmp_path / "h.txt"
src.write_text("header check", encoding="utf-8")
out = loaded.encrypt_file(str(src), USER_PWD)
assert open(out, encoding="utf-8").read().startswith(MAGIC.decode())
def test_default_suffixes(self, loaded, tmp_path):
src = tmp_path / "note.txt"
src.write_text("hello file", encoding="utf-8")
@@ -351,9 +405,14 @@ class TestErrorClass:
# ---------------------------------------------------------------------------
class TestLegacyCompatibility:
def test_format_matches_original_chain(self, loaded):
"""Reproduce the original decryption chain step by step."""
"""Reproduce the original decryption chain step by step.
The artefact carries a header, so the payload is taken from it first;
the chain that follows is the original one, unchanged.
"""
from chaoscrypt._header import strip_header
text = "compat check compatibility"
ct = loaded.encrypt(text, USER_PWD)
ct = strip_header(loaded.encrypt(text, USER_PWD))
core = loaded.core
combined = core._xor_decrypt_with_final_key(ct, USER_PWD)
@@ -370,11 +429,12 @@ class TestLegacyCompatibility:
assert base64.b64decode((sub + "=" * eq_count).encode()).decode() == text
def test_keyfile_readable_by_original_scheme(self, keyfile):
"""A newly generated key file must satisfy the original format."""
"""The payload of a new key file must satisfy the original format."""
from chaoscrypt._header import strip_header
_, path = keyfile
core = ChaosCryptCore()
with open(path, "r") as f:
enc = f.read()
enc = strip_header(f.read())
decrypted = core._xor_decrypt_data(enc, KEY_PWD)
assert decrypted is not None
@@ -462,9 +522,10 @@ class TestCLI:
)
def test_version(self):
from chaoscrypt import __version__
r = self._run(["--version"])
assert r.returncode == 0
assert "2.1.6" in r.stdout
assert __version__ in r.stdout
def test_help_mentions_scope(self):
r = self._run(["--help"])
@@ -574,13 +635,22 @@ class TestCLI:
assert r.stdout.strip() == "", args
def test_plain_output_is_bare_ciphertext(self, tmp_path):
"""Without --json the ciphertext goes to stdout on its own."""
"""Without --json the artefact goes to stdout on its own.
"On its own" means no JSON envelope around it. The artefact does contain
the word ``ciphertext`` inside its header's ``kind`` field, so the check
is for the envelope rather than for the word.
"""
from chaoscrypt._header import MAGIC, strip_header
kf = str(tmp_path / "p.key")
self._run(["keygen", "-k", kf, "-P", KEY_PWD])
r = self._run(["encrypt", "x", "-p", USER_PWD, "-k", kf, "-P", KEY_PWD])
out = r.stdout.strip()
assert out and "ciphertext" not in out
int(out, 16)
assert out
assert not out.startswith("{"), "unexpected JSON envelope"
assert out.startswith(MAGIC.decode())
int(strip_header(out), 16)
def test_chpwd(self, tmp_path):
kf = str(tmp_path / "c.key")
@@ -598,12 +668,17 @@ class TestCLI:
# ---------------------------------------------------------------------------
class TestCoreDirect:
def test_core_generate_and_use(self):
"""Key material moves between instances through the key file scheme."""
from chaoscrypt._header import build_legacy_header
c = ChaosCryptCore()
data = c.generate_keymaterial()
ct = c.encrypt_text("core direct", USER_PWD)
assert c.decrypt_text(ct, USER_PWD) == "core direct"
text = c._xor_encrypt_data(c._obfuscate_keys(data), KEY_PWD)
# load_keys_from_text now requires a header, as every artefact does.
# A minimal one is enough when the material is already at hand.
text = build_legacy_header(is_key=True) + \
c._xor_encrypt_data(c._obfuscate_keys(data), KEY_PWD)
c2 = ChaosCryptCore()
c2.load_keys_from_text(text, KEY_PWD)
assert c2.decrypt_text(ct, USER_PWD) == "core direct"