Fix deverbosification of strings

- Make --disablecompression work
- Support strings containing "⸿" or "¶"
- Don't quote strings containing "\r", "\n" or "”"
- Don't quote strings only containing multiple extended characters
- Properly escape strings containing "´"
- Count ISO-8859-1 extended characters correctly

Author: urkerab

charcoal.py

@@ -20,7 +20,7 @@
     UnicodeLookup, ReverseLookup, UnicodeCommands, InCodepage, sOperator,
     rCommand
 )
-from compression import Decompressed, Compressed
+from compression import Decompressed, Escaped
 from wolfram import *
 from extras import *
 from enum import Enum
@@ -5017,9 +5017,7 @@ def print_xxd(data):
         del raw_file_output
         del file_output
     if argv.disablecompression:
-        StringifierProcessor[CT.String][0] = lambda result: [re.sub(
-            "\n", "¶", rCommand.sub(r"´\1", result[0])
-        )]
+        StringifierProcessor[CT.String][0] = lambda r: [("s", Escaped(r[0])), ("!", "s")]
     if argv.verbose or argv.deverbosify:
         code = ParseExpression(
             code, grammars=VerboseGrammars, processor=StringifierProcessor,
@@ -5057,7 +5055,7 @@ def charcoal_length(character):
             return 5
         length = 0
         for character in code:
-            if InCodepage(character) or ord(character) < 256:
+            if InCodepage(character):
                 length += 1
             else:
                 length += charcoal_length(

codepage.py

@@ -83,8 +83,8 @@ def InCodepage(character):
 UnicodeCommands = "ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ\
 ⁰¹²³⁴⁵⁶⁷⁸⁹αβγδεζηθικλμνξπρσςτυφχψω⟦⟧⦃⦄«»⁺⁻×÷∕﹪∧∨¬⁼‹›＆｜～←↑→↓↖↗↘↙\
 ↶↷⟲¿‽‖·¤¦“”⎚…§⎇↥↧⌊⌈±⊞⊟➙⧴″‴？⪫⪪⌕℅◧◨⮌≡№≔≕▷▶✂⊙⬤✳�≦≧ⅈⅉ；ΣΠ⊕⊖⊗⊘⭆↨⍘₂↔Φ"
-UnicodeCommandRegex = "Ａ-Ｚ⁰¹²³⁴-⁹α-ξπ-ω⟦⟧⦃⦄«»⁺⁻×÷∕﹪∧∨¬⁼‹›＆｜～←-↓↖-↙\
-↶↷⟲¿‽‖·¤¦“”⎚…§⎇↥↧⌊⌈±⊞⊟➙⧴″‴？⪫⪪⌕℅◧◨⮌≡№≔≕▷▶✂⊙⬤✳�≦≧ⅈⅉ；ΣΠ⊕⊖⊗⊘⭆↨⍘₂↔Φ"
+UnicodeCommandRegex = "Ａ-Ｚ⸿¶⁰¹²³⁴-⁹α-ξπ-ω⟦⟧⦃⦄«»⁺⁻×÷∕﹪∧∨¬⁼‹›＆｜～←-↓↖-↙\
+↶↷⟲¿‽‖´·¤¦“”⎚…§⎇↥↧⌊⌈±⊞⊟➙⧴″‴？⪫⪪⌕℅◧◨⮌≡№≔≕▷▶✂⊙⬤✳�≦≧ⅈⅉ；ΣΠ⊕⊖⊗⊘⭆↨⍘₂↔Φ"
 sCommand = "[%s]" % UnicodeCommandRegex
 sOperator = """\
 [ＳＮ‽¬Ｉ‽Ｖ⊟➙⧴″‴↧↥⌊⌈℅⮌⁺⁻×÷∕﹪⁼‹›＆｜～…Ｘ§？⪫⪪⌕◧◨№⎇Ｅ∧∨▷≕✂⊙⬤ⅈⅉ；ΣΠ⊕⊖⊗⊘⭆↨⍘₂↔Φ]\

compression.py

@@ -30,7 +30,20 @@
 LZMA_ENCODING = 125
 
 
-def Compressed(string, escape=False):
+def Escaped(string):
+    """
+    Escaped(string) -> str
+    Returns a Charcoal escaped literal of the given string.
+
+    """
+    if not string:
+        return "””"
+    string = rCommand.sub("´\\1", string)
+    if string[0] in "+X*|-\\/<>^KLTVY7¬":
+        string = "´" + string
+    return re.sub("\r", "⸿", re.sub("\n", "¶", string))
+
+def Compressed(string):
     """
     Compressed(string) -> str
     Returns the shortest Charcoal compressed literal of the given string.
@@ -39,42 +52,29 @@ def Compressed(string, escape=False):
     if not string:
         return "””"
     if not all(
-        character == "¶" or character == "⸿" or
+        character == "\n" or character == "\r" or
         character >= " " and character <= "~"
         for character in string
     ):
-        if not escape:
-            return string
-        if len(re.findall("[^ -~¶⸿]", string)) > 3:
-            return "”" + Codepage[RAW_ENCODING] + string + "”"
-        return (
-            "´" * (string[0] in "+X*|-\\/<>^KLTVY7¬") + 
-            rCommand.sub("´\\1", string)
-        )
-    original_string, string = string, re.sub(
-        "¶", "\n", re.sub("⸿", "\r", string)
-    )
+        if "\n" in string or "\r" in string or "”" in string:
+            return Escaped(string)
+        if len(rCommand.findall(string)) < 3:
+            return Escaped(string)
+        return "”" + Codepage[RAW_ENCODING] + string + "”"
     compressed_charset = CompressCharset(string)
     compressed_rle = CompressRLE(string)
     compressed_brotli = CompressBrotli(string)
     compressed_lzma = CompressLZMA(string)
     compressed_permuted = CompressPermutations(string)
     compressed = CompressString(string)
-    string_length = len(original_string) - 2
+    string_length = len(string) - 2
     minimum_length = min(
         len(compressed_charset), len(compressed_rle), len(compressed_brotli),
         len(compressed_lzma), len(compressed_permuted), len(compressed),
         string_length
     )
     if string_length == minimum_length:
-        if not escape:
-            return original_string
-        if len(re.findall("[^ -~¶⸿]", original_string)) > 3:
-            return "”" + Codepage[RAW_ENCODING] + original_string + "”"
-        return (
-            "´" * (original_string[0] in "+X*|-\\/<>^KLTVY7¬") +
-            rCommand.sub("´\\1", original_string)
-        )
+        return Escaped(string)
     if len(compressed) == minimum_length:
         return "“" + compressed + "”"
     if len(compressed_permuted) == minimum_length:

stringifierprocessor.py

@@ -1,7 +1,6 @@
 from charcoaltoken import CharcoalToken as CT
 from unicodegrammars import UnicodeGrammars
 from compression import Compressed
-from codepage import rCommand
 import re
 
 def string(s):
@@ -14,7 +13,7 @@ def string(s):
     if s == " !\"#$%&'()*+,-./0123456789:;<=>?@\
 ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~":
         return [("v", "γ")]
-    s = Compressed(s.replace("\n", "¶").replace("\r", "⸿"), True)
+    s = Compressed(s)
     if s[0] != "“" and s[0] != "”":
         return [("s", s), ("!", "s")]
     return [("c", s)]

test.py

@@ -1797,6 +1797,7 @@ def test_dictionary_verbose(self):
     def test_compression(self):
         self.assertEqual(Run("”y≔⟦³¦²¦¹⟧β▷sβ”"), "≔⟦³¦²¦¹⟧β▷sβ")
         self.assertEqual(Run("”y≔⟦³¦²¦¹⟧β▷sβ"), "≔⟦³¦²¦¹⟧β▷sβ")
+        self.assertEqual(Run("Print('´⸿¶')", verbose=True), "´⸿¶")
         self.assertEqual(Run("\
 Print('zzyzyzyzyzyzyzyzzzzzzzzyzyz')", verbose=True), "\
 zzyzyzyzyzyzyzyzzzzzzzzyzyz")