lani/fxl.codes.kisekae
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More documentation I don't want to lose and some code cleanup as I understand LZH better...er?

Browse Source
This commit is contained in:
Lani Aung
2024-09-22 16:22:04 -06:00
parent 3425a28bf7
commit d39e73a6fa
16 changed files with 5852 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fxl.codes.kisekae.data.test/GlobalUsings.cs
+1
View File
@@ -0,0 +1 @@
global using NUnit.Framework;
fxl.codes.kisekae.data.test/LhMethodTests.cs
+15
View File
@@ -0,0 +1,15 @@
namespace fxl.codes.kisekae.data.test;
public class LhMethodTests
{
[SetUp]
public void Setup()
{
}
[Test]
public void Test1()
{
Assert.Pass();
}
}
fxl.codes.kisekae.data.test/fxl.codes.kisekae.data.test.csproj
+26
View File
@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/>
<PackageReference Include="NUnit" Version="3.13.3"/>
<PackageReference Include="NUnit3TestAdapter" Version="4.2.1"/>
<PackageReference Include="NUnit.Analyzers" Version="3.6.1"/>
<PackageReference Include="coverlet.collector" Version="6.0.0"/>
</ItemGroup>
<ItemGroup>
<Reference Include="Microsoft.Extensions.DependencyInjection.Abstractions">
<HintPath>..\..\..\..\..\usr\local\share\dotnet\shared\Microsoft.AspNetCore.App\8.0.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll</HintPath>
</Reference>
</ItemGroup>
</Project>
fxl.codes.kisekae.data/Archives/Algorithms/HuffmanDynamic.cs
+5
View File
@@ -0,0 +1,5 @@
namespace fxl.codes.kisekae.data.Archives.Algorithms;
public class HuffmanDynamic : IAlgorithm
{
}
fxl.codes.kisekae.data/Archives/Algorithms/HuffmanStatic.cs
+5
View File
@@ -0,0 +1,5 @@
namespace fxl.codes.kisekae.data.Archives.Algorithms;
public class HuffmanStatic : IAlgorithm
{
}
fxl.codes.kisekae.data/Archives/Algorithms/IAlgorithm.cs
+5
View File
@@ -0,0 +1,5 @@
namespace fxl.codes.kisekae.data.Archives.Algorithms;
public interface IAlgorithm
{
}
fxl.codes.kisekae.data/Archives/EndianUtility.cs
+58
View File
@@ -0,0 +1,58 @@
using System.Buffers.Binary;
namespace fxl.codes.kisekae.data.Archives;
internal static class EndianUtility
{
public static int ToLittleEndian(this ReadOnlySpan<byte> bytes)
{
var x = BitConverter.ToInt32(bytes);
return BitConverter.IsLittleEndian ? x : BinaryPrimitives.ReverseEndianness(x);
}
public static int ReadLittleEndian(this Stream stream, int length = 4)
{
var buffer = new byte[length];
stream.ReadExactly(buffer);
return ToLittleEndian(buffer);
}
public static async Task<int> ReadLittleEndianAsync(this Stream stream, int length = 4)
{
var buffer = new byte[length];
await stream.ReadExactlyAsync(buffer);
return ToLittleEndian(buffer);
}
public static DateTime ToDateTime(this ReadOnlySpan<byte> bytes)
{
var time = ToLittleEndian(bytes);
return ToDateTime(time);
}
private static DateTime ToDateTime(int time)
{
var year = ((time >> 25) & 0x7f) + 1980;
var month = (time >> 21) & 0x0f;
var day = (time >> 16) & 0x1f;
var hour = (time >> 11) & 0x1f;
var minute = (time >> 5) & 0x3f;
var second = (time & 0x1f) * 2;
if (month is < 1 or > 12 || day is < 1 or > 31 || hour >= 24 || minute >= 60 || second > 60) return DateTime.MinValue;
return new DateTime(year, month, day, hour, minute, second);
}
public static DateTime ReadLittleEndianDateTime(this Stream stream)
{
var time = ReadLittleEndian(stream);
return ToDateTime(time);
}
public static async Task<DateTime> ReadLittleEndianDateTimeAsync(this Stream stream)
{
var time = await ReadLittleEndianAsync(stream);
return ToDateTime(time);
}
}
fxl.codes.kisekae.data/Archives/LhCodecService.cs
+36
View File
@@ -0,0 +1,36 @@
using fxl.codes.kisekae.data.Archives.LhHeaders;
using fxl.codes.kisekae.data.Entities;
using Microsoft.Extensions.Logging;
namespace fxl.codes.kisekae.data.Archives;
public class LhCodecService(ILogger<LhCodecService> logger)
{
public async Task<Kisekae?> ReadArchive(Stream stream)
{
try
{
var header = new LhHeader(in stream);
switch (header.MethodId)
{
case "-lh0-": break;
case "-lzs-": break;
case "-lz4-": break;
case "-lh1-": break;
case "-lh2-": break;
case "-lh3-": break;
case "-lh4-": break;
case "-lh5-": break;
case "-lh6-": break;
case "-lh7-": break;
default: throw new Exception("Unknown method");
}
}
catch (Exception e)
{
logger.LogError(e, "Unable to open archive");
}
return null;
}
}
fxl.codes.kisekae.data/Archives/LhHeaders/LhHeader.cs
+153
View File
@@ -0,0 +1,153 @@
using System.Text;
namespace fxl.codes.kisekae.data.Archives.LhHeaders;
internal class LhHeader
{
public LhHeader(ref readonly Stream stream)
{
var bytes = new byte[21];
stream.Position = 0;
stream.ReadExactly(bytes);
var span = bytes.AsSpan();
Level = span[20];
MethodId = Encoding.ASCII.GetString(span[2..7]);
CompressedSize = EndianUtility.ToLittleEndian(span[7..11]);
UncompressedSize = EndianUtility.ToLittleEndian(span[11..15]);
Created = EndianUtility.ToDateTime(span[15..19]);
FileOrDirectory = span[19];
switch (Level)
{
case 0:
Level0(span, in stream);
break;
case 1:
Level1(span, in stream);
break;
case 2:
Level2(span, in stream);
break;
default: throw new Exception("Invalid LH Header");
}
}
public int HeaderSize { get; private set; }
public int HeaderCheckSum { get; private set; }
public string MethodId { get; private set; }
public int CompressedSize { get; private set; }
public int UncompressedSize { get; private set; }
public DateTime Created { get; private set; }
public int FileOrDirectory { get; private set; }
public int Level { get; }
public short Crc16 { get; private set; }
public string? FileName { get; private set; }
public long FileDataPosition { get; private set; }
private void Level0(ReadOnlySpan<byte> span, ref readonly Stream stream)
{
HeaderSize = span[0];
HeaderCheckSum = span[1];
stream.Position = span.Length;
FileName = GetFilename(in stream);
Crc16 = (short)stream.ReadLittleEndian(2);
FileDataPosition = stream.Position;
}
private void Level1(ReadOnlySpan<byte> span, ref readonly Stream stream)
{
Level0(span, in stream);
OperatingSystem = (char)stream.ReadByte();
SetExtendedHeaders(in stream);
FileDataPosition = stream.Position;
}
private void Level2(ReadOnlySpan<byte> span, ref readonly Stream stream)
{
HeaderSize = span[..2].ToLittleEndian();
Crc16 = (short)stream.ReadLittleEndian(2);
OperatingSystem = (char)stream.ReadByte();
SetExtendedHeaders(in stream);
FileDataPosition = stream.Position;
}
private static string GetFilename(ref readonly Stream stream)
{
var length = stream.ReadByte();
var buffer = new byte[length];
stream.ReadExactly(buffer);
return Encoding.ASCII.GetString(buffer);
}
private void SetExtendedHeaders(ref readonly Stream stream)
{
var extendedHeaderSize = stream.ReadLittleEndian(2);
while (extendedHeaderSize != 0)
{
var type = stream.ReadByte();
var nextSize = stream.ReadLittleEndian(2);
switch (type)
{
case 0x00:
Crc16 = (short)stream.ReadLittleEndian(extendedHeaderSize);
break;
case 0x01:
FileName = GetString(in stream, extendedHeaderSize);
break;
case 0x02:
DirectoryName = GetString(in stream, extendedHeaderSize);
break;
case 0x3f:
Comment = GetString(in stream, extendedHeaderSize);
break;
case 0x50:
FilePermission = stream.ReadLittleEndian(extendedHeaderSize);
break;
case 0x51:
GroupId = stream.ReadLittleEndian(2);
UserId = stream.ReadLittleEndian(2);
break;
case 0x52:
GroupName = GetString(in stream, extendedHeaderSize);
break;
case 0x53:
UserName = GetString(in stream, extendedHeaderSize);
break;
case 0x54:
LastModified = stream.ReadLittleEndianDateTime();
break;
default: // Unknown, skip
stream.Position += extendedHeaderSize;
break;
}
extendedHeaderSize = nextSize;
}
}
private static string GetString(ref readonly Stream stream, int length)
{
var buffer = new byte[length];
stream.ReadExactly(buffer);
return Encoding.ASCII.GetString(buffer);
}
#region Extended Headers
public char? OperatingSystem { get; private set; }
public string? DirectoryName { get; private set; }
public string? Comment { get; private set; }
public int? FilePermission { get; private set; }
public int? GroupId { get; private set; }
public string? GroupName { get; private set; }
public int? UserId { get; private set; }
public string? UserName { get; private set; }
public DateTime? LastModified { get; private set; }
#endregion
}
fxl.codes.kisekae.data/Archives/LhMethods/BaseLhMethod.cs
+29
View File
@@ -0,0 +1,29 @@
using fxl.codes.kisekae.data.Archives.Algorithms;
using fxl.codes.kisekae.data.Archives.LhHeaders;
namespace fxl.codes.kisekae.data.Archives.LhMethods;
internal abstract class BaseLhMethod(ref readonly Stream stream, LhHeader header, int slidingWindowBytes, int matchingWindowBytes, IAlgorithm? algorithm)
{
public int SlidingWindow => slidingWindowBytes * 1024;
}
internal class Lzs(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 2, 17, null);
internal class Lz4(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 0, 0, null);
internal class Lh0(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 0, 0, null);
internal class Lh1(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 4, 60, new HuffmanDynamic());
internal class Lh2(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 8, 256, new HuffmanDynamic());
internal class Lh3(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 8, 256, new HuffmanStatic());
internal class Lh4(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 4, 256, new HuffmanStatic());
internal class Lh5(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 8, 256, new HuffmanStatic());
internal class Lh6(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 32, 256, new HuffmanStatic());
internal class Lh7(ref readonly Stream stream, LhHeader header) : BaseLhMethod(in stream, header, 64, 256, new HuffmanStatic());
fxl.codes.kisekae.data/Docs/A Closer Look at Huffman Encoding Using C#.html
+917
View File
@@ -0,0 +1,917 @@
<!DOCTYPE html>
<!-- https://www.carlbelle.com/Articles/Article/2016/Apr/6/ACloserLookAtHuffmanEncodingUsingCSharp/95 -->
<html>
<head>
<title>A Closer Look at Huffman Encoding Using C# | Articles - carlbelle.com</title>
<meta content="Articles about A Closer Look at Huffman Encoding Using C#." name="description"/>
<meta content="" name="keywords"/>
<meta charset="utf-8"/>
<meta content="IE=edge" http-equiv="X-UA-Compatible"/>
<meta content="width=device-width, initial-scale=1" name="viewport"/>
<link href="/Css/Version1.css" media="screen" rel="stylesheet" type="text/css"/>
<link href="/favicon.ico" rel="shortcut icon" type="image/x-icon">
</head>
<body>
<div class="LayoutContainer">
<div class="Header">
<div class="HeaderImage"><a href="/" title="Home - carlbelle.com">CARL.BELLE</a></div>
<div class="HeaderLinks">
<p>
<a class="NoUnderline" href="https://www.github.com/carlyman77" target="_blank" title="GitHub"><span
class="Symbol">&#XE237;</span></a>
<a class="NoUnderline" href="https://www.linkedin.com/in/carlyman77" target="_blank"
title="LinkedIn"><span class="Symbol">&#XE252;</span></a>
<a class="NoUnderline" href="https://www.twitter.com/carlbelle" target="_blank" title="Twitter"><span
class="Symbol">&#XE286;</span></a>
<a class="NoUnderline" href="https://instagram.com/carlyman77" target="_blank" title="Instagram"><span
class="Symbol">&#XE300;</span></a>
<a class="NoUnderline" href="https://open.spotify.com/artist/2kEspNx7mSo6Oe8BWpkGpj" target="_blank"
title="Spotify"><span class="Symbol">&#XE279;</span></a>
<a class="NoUnderline" href="https://www.youtube.com/channel/UCEen0eKme7kKca_yDS7gDzw" target="_blank"
title="SoundCloud"><span class="Symbol">&#XE299;</span></a>
<a class="NoUnderline" href="https://soundcloud.com/carl-belle" target="_blank" title="SoundCloud"><span
class="Symbol">&#XE278;</span></a>
</p>
<p>
<a href="/" title="Home">Home</a> &raquo;
<a href="/Articles/Home" title="Articles">Articles</a> &raquo;
<a href="/QSharp" title="QSharp">QSharp</a> &raquo;
<a href="/Home/Contact" title="Contact">Contact</a>
</p>
</div>
</div>
<div class="HorizontalSolidLine"></div>
<div class="HorizontalSpacer"></div>
<div class="Layout">
<h1>A Closer Look at Huffman Encoding Using C#</h1>
<p>06 Apr 2016
<p>
<p>
<a href="/Articles/Category/Algorithms/18" title="Algorithms">Algorithms</a>, <a
href="/Articles/Category/CSharp/11" title="C#">C#</a></p>
<br/>
<p>On this page:</p>
<ul>
<li><a href="#HowItWorks">How it Works</a></li>
<li><a href="#ConstructingTheBinaryTree">Constructing the Binary Tree</a></li>
<li><a href="#EncodingTheData">Encoding the Data</a></li>
<li><a href="#UnpackingTheData">Unpacking the Data</a></li>
<li><a href="#FinalThoughts">Final Thoughts</a></li>
<li><a href="#TheCode">The Code</a></li>
</ul>
<p>Huffman encoding is a compression technique developed by David Huffman and published in his 1952 paper 'A
Method for the Construction of Minimum-Redundancy Codes'. The technique centres on encoding characters as a
list of code words, which are then used to generate a coded version of the original data. Huffman encoding
can be very efficient, and should generally yield compression as low as 20% of the original data size.</p>
<a name="#HowItWorks"></a>
<p><strong>How it Works</strong></p>
<p>The input data is scanned, and a list of characters and their corresponding weight (the number of instances
the character appears in the data). The weights of each character are used to compute the length of the code
words that represent that character: larger weights should yield smaller code words, a key aspect of
compressing the data. Characters are then arranged into a binary tree, where the resulting layout is used to
determine the encoding table. The last step is using the encoding table to create the new file.</p>
<a name="#ConstructingTheBinaryTree"></a>
<p><strong>Constructing the Binary Tree</strong></p>
<p>Consider the following text:</p>
<pre>
abccccddddddefghijjjjj
</pre>
<p>Represented as a list of characters with corresponding weights:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
0 a 1
1 b 1
2 c 4
3 d 6
4 e 1
5 f 1
6 g 1
7 h 1
8 i 1
9 j 5
</pre>
<!-- END AUTO FORMAT -->
<p>This list is now turned into a binary tree. A binary tree is a tree where each node may have a maximum of two
child nodes. The left child node is consider the 0 child node, whereas the right child node is considered
the 1 child node. The first two nodes are removed from the list, and a new parent node is constructed. This
parent node's weight is the combined weight of both children. In our example, after the first iteration our
list would look like this:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
00 null null <span class="Comment">// used to be a</span>
01 null null <span class="Comment">// used to be b</span>
02 e 1
03 f 1
04 g 1
05 h 1
06 i 1
07 c 4
08 j 5
09 d 6
10 a + b 2 <span class="Comment">// new parent node</span>
</pre>
<!-- END AUTO FORMAT -->
<p>The next two lowest weight nodes are added to a parent:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
00 null null <span class="Comment">// used to be a</span>
01 null null <span class="Comment">// used to be b</span>
02 null null <span class="Comment">// used to be e</span>
03 null null <span class="Comment">// used to be f</span>
04 g 1
05 h 1
06 i 1
07 c 4
08 j 5
09 d 6
10 a + b 2 <span class="Comment">// new parent node</span>
11 e + f 2 <span class="Comment">// new parent node</span>
</pre>
<!-- END AUTO FORMAT -->
<p>And so on. At the end of this round of processing, the list looks like this:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
00 null null <span class="Comment">// used to be a</span>
01 null null <span class="Comment">// used to be b</span>
02 null null <span class="Comment">// used to be e</span>
03 null null <span class="Comment">// used to be f</span>
04 null null <span class="Comment">// used to be g</span>
05 null null <span class="Comment">// used to be h</span>
06 null null <span class="Comment">// used to be i</span>
07 null null <span class="Comment">// used to be c</span>
08 null null <span class="Comment">// used to be j</span>
09 null null <span class="Comment">// used to be d</span>
10 a + b 2 <span class="Comment">// new parent node</span>
11 e + f 2 <span class="Comment">// new parent node</span>
12 g + h 2 <span class="Comment">// new parent node</span>
13 i + c 5 <span class="Comment">// new parent node</span>
14 j + d 11 <span class="Comment">// new parent node</span>
</pre>
<!-- END AUTO FORMAT -->
<p>Null nodes are removed from the list:</p>
<pre>
Index Char Weight
0 a + b 2
1 e + f 2
2 g + h 2
3 i + c 5
4 j + d 11
</pre>
<p>And the next round of processing begins, always taking the smallest weighted nodes and combining them under a
new parent. At the end of the next round of processing, the list looks like this:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
0 null null <span class="Comment">// a + b</span>
1 null null <span class="Comment">// e + f</span>
2 null null <span class="Comment">// g + h</span>
3 null null <span class="Comment">// i + c</span>
4 j + d 11
5 a + b (2), e + f (2) 4 <span class="Comment">// new parent node</span>
6 g + h (2), i + c (5) 7 <span class="Comment">// new parent node</span>
</pre>
<!-- END AUTO FORMAT -->
<p>Null nodes are removed from the list:</p>
<pre>
Index Char Weight
0 a + b (2), e + f (2) 4
1 g + h (2), i + c (5) 7
2 j + d 11
</pre>
<p>At the end of the next round of processing:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
0 null 4 <span class="Comment">// a + b (2), e + f (2)</span>
1 null 7 <span class="Comment">// g + h (2), i + c (5) </span>
2 j + d 11 <span class="Comment">// new parent node</span>
3 (a + b, e + f) - (4), (g + h, i + c) - (7) 11 <span class="Comment">// new parent node</span>
</pre>
<!-- END AUTO FORMAT -->
<p>After removing null nodes from the list, we are ready for the final round of processing:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
0 j + d 11 <span class="Comment">// new parent node</span>
1 (a + b, e + f) - (4), (g + h, i + c) - (7) 11 <span class="Comment">// new parent node</span>
</pre>
<!-- END AUTO FORMAT -->
<p>And finally, after all processing, there is only one node left:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char Weight
0 (j + d) - 11, (a + b, e + f, g + h, i + c) - (11) 22 <span class="Comment">// new parent node</span>
</pre>
<!-- END AUTO FORMAT -->
<p>This final node forms the root of the tree, and is used to generate a list of binary words that represent
each character. Starting at the root of the tree, following a left child node inserts a 0 into the binary
word, following a right child node inserts a 1 into the binary word. Using this method, the following list
of binary words is created:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Char Weight BinaryWord Int32Word
j 5 00 0 <span class="Comment">// This column is for sorting purposes only</span>
d 6 01 1
a 1 1000 8
b 1 1001 9
e 1 1010 10
f 1 1011 11
g 1 1100 12
h 1 1101 13
I 1 1110 14
c 4 1111 15
</pre>
<!-- END AUTO FORMAT -->
<p>Note that nodes without a character value are not present in this list, but are able to be inferred from the
list sorting operations. For the moment, I'll leave you to construct the tree using the information
above.</p>
<a name="#EncodingTheData"></a>
<p><strong>Encoding the Data</strong></p>
<p>The next step is to encode the original data using the binary words created from the previous steps. Moving
forward through the data one byte at a time, each character is used to perform a lookup for its
corresponding binary word. This binary word is added into a binary word buffer. When the binary word buffer
reaches sufficient length, 8 values are read off and removed to construct a new byte. Consider the
following:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Char BinaryWord ByteBuffer
00 a 1000 <span class="Comment">// Only 4 bits here, not enough for a new byte</span>
01 b 1001 <span class="Comment">// Combined with the previous entry, there are enough bits here for a new byte: 10001001 => 137, remove 8 entries from the buffer</span>
02 c 1111 <span class="Comment">// Start a new byte using these 4 bits</span>
03 c 1111 <span class="Comment">// 11111111 => 255</span>
04 c 1111 <span class="Comment">// Start a new byte</span>
05 c 1111 <span class="Comment">// 11111111 => 255</span>
06 d 01 <span class="Comment">// Start a new byte</span>
07 d 01 <span class="Comment">// 0101 - the buffer is only half full</span>
08 d 01 <span class="Comment">// 010101</span>
09 d 01 <span
class="Comment">// 01010101 => 85 - note that this one byte actually represents 4 chars!</span>
10 d 01 <span class="Comment">// Start a new byte</span>
11 d 01 <span class="Comment">// 0101</span>
12 e 1010 <span class="Comment">// 01011010 => 90 - this one byte represents 3 chars</span>
13 f 1011 <span class="Comment">// Start a new byte</span>
14 g 1100 <span class="Comment">// 10111100 => 188</span>
15 h 1101 <span class="Comment">// New byte</span>
16 i 1110 <span class="Comment">// 11011110 => 222</span>
17 j 00 <span class="Comment">// New byte</span>
18 j 00 <span class="Comment">// 0000</span>
19 j 00 <span class="Comment">// 000000</span>
20 j 00 <span class="Comment">// 00000000 => 0</span>
21 j 00 <span class="Comment">// Start a new byte</span>
</pre>
<!-- END AUTO FORMAT -->
<p>With the processing of the final character, the byte buffer only contains 2 values, which is not enough to
construct a byte, however as the binary word is 00 it is simply converted to 0. Therefore, the final byte to
be constructed is 0. Processing of these 22 characters has yielded a list containing only 9 bytes, about 41%
of the original data size.</p>
<a name="#UnpackingTheData"></a>
<p><strong>Unpacking the Data</strong></p>
<p>Decompression (in this example at least) is relatively straight forward. The byte list is read, and each byte
converted into a binary word, which is added into a buffer. If the buffer is less than 8 values, then it is
'fixed' (more on that later). Reading each bit from the binary word indicates which path to take through the
binary tree (constructed above): starting at the root node, follow the left child for a 0 value, and follow
the right for a 1 value. The entire decompression process for the above data:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
Index Byte Characters
0 137 =&gt; 10001001 a, b
1 255 =&gt; 11111111 c, c
2 255 =&gt; 11111111 c, c
3 85 =&gt; 01010101 d, d, d, d
4 90 =&gt; 01011010 d, d, e
5 188 =&gt; 10111100 f, g
6 222 =&gt; 11011110 h, i
7 0 =&gt; 00000000 j, j, j, j
8 0 => 00 j <span class="Comment">// This is the last value, so rather than pad it out, we can just go directly to the char value</span>
</pre>
<!-- END AUTO FORMAT -->
<p>The output data now contains 22 characters, the data has been successfully unpacked.</p>
<a name="#FinalThoughts"></a>
<p><strong>Final Thoughts</strong></p>
<p>Conveniently, this example ignores completely the size of the encoding table, as well as how storage and
retrieval mechanisms, so theoretically 9 bytes is probably as small as we can get. If this compressed data
were to be used outside this example, then the binary tree and code words would also need to be stored along
with the byte list in order to unpack the data at some other time.</p>
<a name="#TheCode"></a>
<p><strong>The Code</strong></p>
<p>I've included my somewhat rambling encoding and decoding process, in C# below.</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
<span class="Keyword">public</span> <span class="Keyword">class</span> <span class="Type">HuffmanCompressor</span>
{
<span class="Keyword">private</span> <span class="Type">HuffmanNode</span> oRootHuffmanNode;
<span class="Keyword">private</span> <span class="Type">List</span>&lt;<span class="Type">HuffmanNode</span>&gt; oValueHuffmanNodes;
<span class="Keyword">private</span> <span class="Type">List</span>&lt;<span class="Type">HuffmanNode</span>&gt; BuildBinaryTree(<span
class="Keyword">string</span> Value)
{
<span class="Type">List</span>&lt;<span class="Type">HuffmanNode</span>&gt; oHuffmanNodes = GetInitialNodeList();
<span class="Comment">// Update node weights</span>
Value.ToList().ForEach(m =&gt; oHuffmanNodes[m].Weight++);
<span class="Comment">// Select only nodes that have a weight</span>
oHuffmanNodes = oHuffmanNodes
.Where(m =&gt; (m.Weight &gt; 0))
.OrderBy(m =&gt; (m.Weight))
.ThenBy(m =&gt; (m.Value))
.ToList();
<span class="Comment">// Assign parent nodes</span>
oHuffmanNodes = UpdateNodeParents(oHuffmanNodes);
oRootHuffmanNode = oHuffmanNodes[0];
oHuffmanNodes.Clear();
<span class="Comment">// Sort nodes into a tree</span>
SortNodes(oRootHuffmanNode, oHuffmanNodes);
<span class="Keyword">return</span> oHuffmanNodes;
}
<span class="Keyword">public</span> <span class="Keyword">void</span> Compress(<span class="Keyword">string</span> FileName)
{
<span class="Type">FileInfo</span> oFileInfo = <span class="Keyword">new</span> <span
class="Type">FileInfo</span>(FileName);
<span class="Keyword">if</span> (oFileInfo.Exists == <span class="Keyword">true</span>)
{
<span class="Keyword">string</span> sFileContents = <span class="Literal">""</span>;
<span class="Keyword">using</span> (<span class="Type">StreamReader</span> oStreamReader = <span
class="Keyword">new</span> <span class="Type">StreamReader</span>(<span class="Type">File</span>.OpenRead(FileName)))
{
sFileContents = oStreamReader.ReadToEnd();
}
<span class="Type">List</span>&lt;<span class="Type">HuffmanNode</span>&gt; oHuffmanNodes = BuildBinaryTree(sFileContents);
<span class="Comment">// Filter to nodes we care about</span>
oValueHuffmanNodes = oHuffmanNodes
.Where(m =&gt; (m.Value.HasValue == <span class="Keyword">true</span>))
.OrderBy(m => (m.BinaryWord)) <span
class="Comment">// Not really required, for presentation purposes</span>
.ToList();
<span class="Comment">// Construct char to binary word dictionary for quick value to binary word resolution</span>
<span class="Type">Dictionary</span>&lt;char, <span class="Keyword">string</span>&gt; oCharToBinaryWordDictionary = <span
class="Keyword">new</span> <span class="Type">Dictionary</span>&lt;char, <span
class="Keyword">string</span>&gt;();
<span class="Keyword">foreach</span> (<span class="Type">HuffmanNode</span> oHuffmanNode <span
class="Keyword">in</span> oValueHuffmanNodes)
{
oCharToBinaryWordDictionary.Add(oHuffmanNode.Value.Value, oHuffmanNode.BinaryWord);
}
<span class="Type">StringBuilder</span> oStringBuilder = <span class="Keyword">new</span> <span
class="Type">StringBuilder</span>();
<span class="Type">List</span>&lt;<span class="Keyword">byte</span>&gt; oByteList = <span class="Keyword">new</span> <span
class="Type">List</span>&lt;<span class="Keyword">byte</span>&gt;();
<span class="Keyword">for</span> (<span class="Keyword">int</span> i = 0; i &lt; sFileContents.Length; i++)
{
<span class="Keyword">string</span> sWord = <span class="Literal">""</span>;
<span class="Comment">// Append the binary word value using the char located at the current file position</span>
oStringBuilder.Append(oCharToBinaryWordDictionary[sFileContents[i]]);
<span class="Comment">// Once we have at least 8 chars, we can construct a byte</span>
<span class="Keyword">while</span> (oStringBuilder.Length &gt;= 8)
{
sWord = oStringBuilder.ToString().Substring(0, 8);
<span class="Comment">// Remove the word to be constructed from the buffer</span>
oStringBuilder.Remove(0, sWord.Length);
}
<span class="Comment">// Convert the word and add it onto the list</span>
<span class="Keyword">if</span> (<span class="Type">String</span>.IsNullOrEmpty(sWord) == <span
class="Keyword">false</span>)
{
oByteList.Add(Convert.ToByte(sWord, 2));
}
}
<span class="Comment">// If there is anything in the buffer, make sure it is retrieved</span>
<span class="Keyword">if</span> (oStringBuilder.Length &gt; 0)
{
<span class="Keyword">string</span> sWord = oStringBuilder.ToString();
<span class="Comment">// Convert the word and add it onto the list</span>
<span class="Keyword">if</span> (<span class="Type">String</span>.IsNullOrEmpty(sWord) == <span
class="Keyword">false</span>)
{
oByteList.Add(Convert.ToByte(sWord, 2));
}
}
<span class="Comment">// Write compressed file</span>
<span class="Keyword">string</span> sCompressedFileName = Path.Combine(oFileInfo.Directory.FullName, <span
class="Type">String</span>.Format(<span class="Literal">"{0}.compressed"</span>, oFileInfo.Name.Replace(oFileInfo.Extension, <span
class="Literal">""</span>)));
<span class="Keyword">if</span> (<span class="Type">File</span>.Exists(sCompressedFileName) == <span
class="Keyword">true</span>)
{
<span class="Type">File</span>.Delete(sCompressedFileName);
}
<span class="Keyword">using</span> (<span class="Type">FileStream</span> oFileStream = <span class="Type">File</span>.OpenWrite(sCompressedFileName))
{
oFileStream.Write(oByteList.ToArray(), 0, oByteList.Count);
}
}
}
<span class="Keyword">public</span> <span class="Keyword">void</span> Decompress(<span class="Keyword">string</span> FileName)
{
<span class="Type">FileInfo</span> oFileInfo = <span class="Keyword">new</span> <span
class="Type">FileInfo</span>(FileName);
<span class="Keyword">if</span> (oFileInfo.Exists == <span class="Keyword">true</span>)
{
<span class="Keyword">string</span> sCompressedFileName = <span class="Type">String</span>.Format(<span
class="Literal">"{0}.compressed"</span>, oFileInfo.FullName.Replace(oFileInfo.Extension, <span
class="Literal">""</span>));
<span class="Keyword">byte</span>[] oBuffer = <span class="Keyword">null</span>;
<span class="Keyword">using</span> (<span class="Type">FileStream</span> oFileStream = <span class="Type">File</span>.OpenRead(sCompressedFileName))
{
oBuffer = <span class="Keyword">new</span> <span class="Keyword">byte</span>[oFileStream.Length];
oFileStream.Read(oBuffer, 0, oBuffer.Length);
}
<span class="Comment">// Find the zero node</span>
<span class="Type">HuffmanNode</span> oZeroHuffmanNode = oRootHuffmanNode;
<span class="Keyword">while</span> (oZeroHuffmanNode.Left != <span class="Keyword">null</span>)
{
oZeroHuffmanNode = oZeroHuffmanNode.Left;
}
<span class="Comment">// Unpack the file contents</span>
<span class="Type">HuffmanNode</span> oCurrentHuffmanNode = <span class="Keyword">null</span>;
<span class="Type">StringBuilder</span> oStringBuilder = <span class="Keyword">new</span> <span
class="Type">StringBuilder</span>();
<span class="Keyword">for</span> (<span class="Keyword">int</span> i = 0; i &lt; oBuffer.Length; i++)
{
<span class="Keyword">string</span> sBinaryWord = <span class="Literal">""</span>;
<span class="Keyword">byte</span> oByte = oBuffer[i];
<span class="Keyword">if</span> (oByte == 0)
{
sBinaryWord = oZeroHuffmanNode.BinaryWord;
}
<span class="Keyword">else</span>
{
sBinaryWord = Convert.ToString(oByte, 2);
}
<span class="Keyword">if</span> ((sBinaryWord.Length &lt; 8) && (i &lt; (oBuffer.Length - 1)))
{
<span class="Comment">// Pad binary word out to 8 places</span>
<span class="Type">StringBuilder</span> oBinaryStringBuilder = <span
class="Keyword">new</span> <span class="Type">StringBuilder</span>(sBinaryWord);
<span class="Keyword">while</span> (oBinaryStringBuilder.Length &lt; 8)
{
oBinaryStringBuilder.Insert(0, <span class="Literal">"0"</span>);
}
sBinaryWord = oBinaryStringBuilder.ToString();
}
<span class="Comment">// Use the binary word to navigate the tree looking for the value</span>
<span class="Keyword">for</span> (<span class="Keyword">int</span> j = 0; j &lt; sBinaryWord.Length; j++)
{
<span class="Keyword">char</span> cValue = sBinaryWord[j];
<span class="Keyword">if</span> (oCurrentHuffmanNode == <span class="Keyword">null</span>)
{
oCurrentHuffmanNode = oRootHuffmanNode;
}
<span class="Keyword">if</span> (cValue == '0')
{
oCurrentHuffmanNode = oCurrentHuffmanNode.Left;
}
<span class="Keyword">else</span>
{
oCurrentHuffmanNode = oCurrentHuffmanNode.Right;
}
<span class="Keyword">if</span> ((oCurrentHuffmanNode.Left == <span class="Keyword">null</span>) && (oCurrentHuffmanNode.Right == <span
class="Keyword">null</span>))
{
<span class="Comment">// No more child nodes to choose from, so this must be a value node</span>
oStringBuilder.Append(oCurrentHuffmanNode.Value.Value);
oCurrentHuffmanNode = <span class="Keyword">null</span>;
}
}
}
<span class="Comment">// Write out file</span>
<span class="Keyword">string</span> sUncompressedFileName = Path.Combine(oFileInfo.Directory.FullName, <span
class="Type">String</span>.Format(<span class="Literal">"{0}.uncompressed"</span>, oFileInfo.Name.Replace(oFileInfo.Extension, <span
class="Literal">""</span>)));
<span class="Keyword">if</span> (<span class="Type">File</span>.Exists(sUncompressedFileName) == <span
class="Keyword">true</span>)
{
<span class="Type">File</span>.Delete(sUncompressedFileName);
}
<span class="Keyword">using</span> (<span class="Type">StreamWriter</span> oStreamWriter = <span
class="Keyword">new</span> <span class="Type">StreamWriter</span>(<span class="Type">File</span>.OpenWrite(sUncompressedFileName)))
{
oStreamWriter.Write(oStringBuilder.ToString());
}
}
}
<span class="Keyword">private</span> <span class="Keyword">static</span> <span class="Type">List</span>&lt;<span
class="Type">HuffmanNode</span>&gt; GetInitialNodeList()
{
<span class="Type">List</span>&lt;<span class="Type">HuffmanNode</span>&gt; oGetInitialNodeList = <span
class="Keyword">new</span> <span class="Type">List</span>&lt;<span class="Type">HuffmanNode</span>&gt;();
<span class="Keyword">for</span> (<span class="Keyword">int</span> i = Char.MinValue; i &lt; Char.MaxValue; i++)
{
oGetInitialNodeList.Add(<span class="Keyword">new</span> <span class="Type">HuffmanNode</span>((<span
class="Keyword">char</span>)(i)));
}
<span class="Keyword">return</span> oGetInitialNodeList;
}
<span class="Keyword">private</span> <span class="Keyword">static</span> <span class="Keyword">void</span> SortNodes(<span
class="Type">HuffmanNode</span> Node, <span class="Type">List</span>&lt;<span
class="Type">HuffmanNode</span>&gt; Nodes)
{
<span class="Keyword">if</span> (Nodes.Contains(Node) == <span class="Keyword">false</span>)
{
Nodes.Add(Node);
}
<span class="Keyword">if</span> (Node.Left != <span class="Keyword">null</span>)
{
SortNodes(Node.Left, Nodes);
}
<span class="Keyword">if</span> (Node.Right != <span class="Keyword">null</span>)
{
SortNodes(Node.Right, Nodes);
}
}
<span class="Keyword">private</span> <span class="Keyword">static</span> <span class="Type">List</span>&lt;<span
class="Type">HuffmanNode</span>&gt; UpdateNodeParents(<span class="Type">List</span>&lt;<span
class="Type">HuffmanNode</span>&gt; Nodes)
{
<span class="Comment">// Assign parent nodes</span>
<span class="Keyword">while</span> (Nodes.Count &gt; 1)
{
<span class="Keyword">int</span> iOperations = (Nodes.Count / 2);
<span class="Keyword">for</span> (<span class="Keyword">int</span> iOperation = 0, i = 0, j = 1; iOperation &lt; iOperations; iOperation++, i += 2, j += 2)
{
<span class="Keyword">if</span> (j &lt; Nodes.Count)
{
<span class="Type">HuffmanNode</span> oParentHuffmanNode = <span class="Keyword">new</span> <span
class="Type">HuffmanNode</span>(Nodes[i], Nodes[j]);
Nodes.Add(oParentHuffmanNode);
Nodes[i] = <span class="Keyword">null</span>;
Nodes[j] = <span class="Keyword">null</span>;
}
}
<span class="Comment">// Remove null nodes</span>
Nodes = Nodes
.Where(m =&gt; (m != <span class="Keyword">null</span>))
.OrderBy(m => (m.Weight)) <span class="Comment">// Choose the lowest weightings first</span>
.ToList();
}
<span class="Keyword">return</span> Nodes;
}
</pre>
<!-- END AUTO FORMAT -->
<p>Here's the code for the HuffmanNode type itself:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
<span class="Keyword">public</span> <span class="Keyword">class</span> <span class="Type">HuffmanNode</span>
{
<span class="Keyword">public</span> <span class="Type">HuffmanNode</span>()
{
}
<span class="Keyword">public</span> <span class="Type">HuffmanNode</span>(<span class="Keyword">char</span> Value)
{
cValue = Value;
}
<span class="Keyword">public</span> <span class="Type">HuffmanNode</span>(<span class="Type">HuffmanNode</span> Left, <span
class="Type">HuffmanNode</span> Right)
{
oLeft = Left;
oLeft.oParent = <span class="Keyword">this</span>;
oLeft.bIsLeftNode = <span class="Keyword">true</span>;
oRight = Right;
oRight.oParent = <span class="Keyword">this</span>;
oRight.bIsRightNode = <span class="Keyword">true</span>;
iWeight = (oLeft.Weight + oRight.Weight);
}
<span class="Keyword">private</span> <span class="Keyword">string</span> sBinaryWord;
<span class="Keyword">private</span> <span class="Keyword">bool</span> bIsLeftNode;
<span class="Keyword">private</span> <span class="Keyword">bool</span> bIsRightNode;
<span class="Keyword">private</span> <span class="Type">HuffmanNode</span> oLeft;
<span class="Keyword">private</span> <span class="Type">HuffmanNode</span> oParent;
<span class="Keyword">private</span> <span class="Type">HuffmanNode</span> oRight;
<span class="Keyword">private</span> <span class="Keyword">char</span>? cValue;
<span class="Keyword">private</span> <span class="Keyword">int</span> iWeight;
<span class="Keyword">public</span> <span class="Keyword">string</span> BinaryWord
{
<span class="Keyword">get</span>
{
<span class="Keyword">string</span> sReturnValue = <span class="Literal">""</span>;
<span class="Keyword">if</span> (<span class="Type">String</span>.IsNullOrEmpty(sBinaryWord) == <span
class="Keyword">true</span>)
{
<span class="Type">StringBuilder</span> oStringBuilder = <span class="Keyword">new</span> <span
class="Type">StringBuilder</span>();
<span class="Type">HuffmanNode</span> oHuffmanNode = <span class="Keyword">this</span>;
<span class="Keyword">while</span> (oHuffmanNode != <span class="Keyword">null</span>)
{
<span class="Keyword">if</span> (oHuffmanNode.bIsLeftNode == <span class="Keyword">true</span>)
{
oStringBuilder.Insert(0, <span class="Literal">"0"</span>);
}
<span class="Keyword">if</span> (oHuffmanNode.bIsRightNode == <span class="Keyword">true</span>)
{
oStringBuilder.Insert(0, <span class="Literal">"1"</span>);
}
oHuffmanNode = oHuffmanNode.oParent;
}
sReturnValue = oStringBuilder.ToString();
sBinaryWord = sReturnValue;
}
<span class="Keyword">else</span>
{
sReturnValue = sBinaryWord;
}
<span class="Keyword">return</span> sReturnValue;
}
}
<span class="Keyword">public</span> <span class="Type">HuffmanNode</span> Left
{
<span class="Keyword">get</span>
{
<span class="Keyword">return</span> oLeft;
}
}
<span class="Keyword">public</span> <span class="Type">HuffmanNode</span> Parent
{
<span class="Keyword">get</span>
{
<span class="Keyword">return</span> oParent;
}
}
<span class="Keyword">public</span> <span class="Type">HuffmanNode</span> Right
{
<span class="Keyword">get</span>
{
<span class="Keyword">return</span> oRight;
}
}
<span class="Keyword">public</span> <span class="Keyword">char</span>? Value
{
<span class="Keyword">get</span>
{
<span class="Keyword">return</span> cValue;
}
}
<span class="Keyword">public</span> <span class="Keyword">int</span> Weight
{
<span class="Keyword">get</span>
{
<span class="Keyword">return</span> iWeight;
}
<span class="Keyword">set</span>
{
iWeight = <span class="Keyword">value</span>;
}
}
<span class="Keyword">public</span> <span class="Keyword">override</span> <span class="Keyword">string</span> ToString()
{
<span class="Type">StringBuilder</span> oStringBuilder = <span class="Keyword">new</span> <span class="Type">StringBuilder</span>();
<span class="Keyword">if</span> (cValue.HasValue == <span class="Keyword">true</span>)
{
oStringBuilder.AppendFormat(<span class="Literal">"'{0}' ({1}) - {2} ({3})"</span>, cValue.Value, iWeight, BinaryWord, BinaryWord.BinaryStringToInt32());
}
<span class="Keyword">else</span>
{
<span class="Keyword">if</span> ((oLeft != <span class="Keyword">null</span>) && (oRight != <span
class="Keyword">null</span>))
{
<span class="Keyword">if</span> ((oLeft.Value.HasValue == <span class="Keyword">true</span>) && (oRight.Value.HasValue == <span
class="Keyword">true</span>))
{
oStringBuilder.AppendFormat(<span class="Literal">"{0} + {1} ({2})"</span>, oLeft.Value, oRight.Value, iWeight);
}
<span class="Keyword">else</span>
{
oStringBuilder.AppendFormat(<span class="Literal">"{0}, {1} - ({2})"</span>, oLeft, oRight, iWeight);
}
}
<span class="Keyword">else</span>
{
oStringBuilder.Append(iWeight);
}
}
<span class="Keyword">return</span> oStringBuilder.ToString();
}
}
</pre>
<!-- END AUTO FORMAT -->
<p>And finally, an extension method that is also required for the code to compile:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
<span class="Keyword">public</span> <span class="Keyword">static</span> <span class="Keyword">int</span> BinaryStringToInt32(<span
class="Keyword">this</span> <span class="Keyword">string</span> Value)
{
<span class="Keyword">int</span> iBinaryStringToInt32 = 0;
<span class="Keyword">for</span> (<span class="Keyword">int</span> i = (Value.Length - 1), j = 0; i &gt;= 0; i--, j++)
{
iBinaryStringToInt32 += ((Value[j] == '0' ? 0 : 1) * (<span class="Keyword">int</span>)(<span
class="Type">Math</span>.Pow(2, i)));
}
<span class="Keyword">return</span> iBinaryStringToInt32;
}
</pre>
<!-- END AUTO FORMAT -->
<p>To get this code running, it will need to be stitched together, perhaps in a console application:</p>
<!-- BEGIN AUTO FORMAT -->
<pre>
<span class="Keyword">public</span> <span class="Keyword">static</span> <span class="Keyword">void</span> Main(<span
class="Keyword">string</span>[] Arguments)
{
<span class="Type">HuffmanCompressor</span> oHuffmanCompressor = <span class="Keyword">new</span> <span
class="Type">HuffmanCompressor</span>();
oHuffmanCompressor.Compress(<span class="Literal">"Huffman.txt"</span>);
<span class="Type">Console</span>.WriteLine();
oHuffmanCompressor.Decompress(<span class="Literal">"Huffman.txt"</span>);
<span class="Type">Console</span>.WriteLine();
}
</pre>
<!-- END AUTO FORMAT -->
<p>Note that for this example, decompression is only possible if the binary tree has been constructed, i,e., the
data has already been compressed.</p>
<br/>
<p>&nbsp;</p>
</div>
<div class="HorizontalSpacer"></div>
<div class="HorizontalSolidLine"></div>
<p align="center">Copyright &copy; 2024 <a href="/" title="carlbelle.com">carlbelle.com</a></p>
<div class="HorizontalSpacer"></div>
</div>
<script type="text/javascript">
var sGuid = '3C9AAA06-9031-4B3E-8AFE-586AE8F6E17F';
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-26910229-1']);
_gaq.push(['_trackPageview']);
(function () {
var ga = document.createElement('script');
ga.type = 'text/javascript';
ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ga, s);
})();
</script>
</body>
</html>
fxl.codes.kisekae.data/Docs/Data Compression Algorithms of LARC and LHarc.html
+374
View File
@@ -0,0 +1,374 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- http://archive.gamedev.net/archive/reference/articles/article295.html -->
<html xmlns:fb="http://www.facebook.com/2008/fbml" xmlns:og="http://opengraphprotocol.org/schema/">
<head>
<title>GameDev.net - Data Compression Algorithms of LARC and LHarc</title>
<meta content="Data Compression Algorithms of LARC and LHarc" property="og:title"/>
<meta content="article" property="og:type"/>
<meta content="http://www.gamedev.net/reference/articles/article295.asp" property="og:url"/>
<meta content="../../../images.gamedev.net/icons/gamedev_net.png" property="og:image"/>
<meta content="GameDev.net" property="og:site_name"/>
<meta content="103355463445" property="fb:app_id"/>
<meta content="Covers a number of compression algorithms, including LZSS, LZW, Huffman, Arithmetic, LZARI, and LZHUF."
property="og:description"/>
<link href="../../css/reference.css" rel=stylesheet type="text/css">
<link href="../../pics/gdicon.png" rel="icon" type="image/png">
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-279474-1']);
_gaq.push(['_trackPageview']);
(function () {
var ga = document.createElement('script');
ga.type = 'text/javascript';
ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ga, s);
})();
</script>
<script src="../../../partner.googleadservices.com/gampad/google_service.js" type="text/javascript"></script>
<script type="text/javascript">
GS_googleAddAdSenseService("ca-pub-3167291168602081");
GS_googleEnableAllServices();
</script>
<script type="text/javascript">
GA_googleAddSlot("ca-pub-3167291168602081", "Leaderboard_Main");
GA_googleAddSlot("ca-pub-3167291168602081", "Box_Main");
</script>
<script type="text/javascript">
GA_googleFetchAds();
</script>
</head>
<body>
<div id="header"><a href="../../index.html">GameDev.net</a>Data Compression Algorithms of LARC and LHarc</div>
<div id="fb-root"></div>
<script>
window.fbAsyncInit = function () {
FB.init({appId: '103355463445', status: true, cookie: true, xfbml: true});
};
(function () {
var e = document.createElement('script');
e.async = true;
e.src = document.location.protocol +
'//connect.facebook.net/en_US/all.js';
document.getElementById('fb-root').appendChild(e);
}());
</script>
<center>
<script type="text/javascript">
GA_googleFillSlot("Leaderboard_Main");
</script>
</center>
<table border="0" cellpadding="5" cellspacing="0" width="100%">
<tr>
<td>
<CENTER><SPAN CLASS="title">Data Compression Algorithms of LARC and LHarc</SPAN>
<BR><SPAN CLASS="author">by Haruhiko Okumura*</SPAN></CENTER>
<P><I>*The author is the sysop of the Science SIG of PV-VAN. His address is: 12-2-404 Green Heights, 580
Nagasawa, Yokosuka 239 Japan</I>
<H1>Introduction</H1>
<P>In the spring of 1988, I wrote a very simple data compression program named LZSS in C language, and
uploaded it to the Science SIG (forum) of PC-VAN, Japan's biggest personal computer network.
<P>That program was based on Storer and Szymanski's slightly modified version of one of Lempel and Ziv's
algorithms. Despite its simplicity, for most files its compression outperformed the archivers then
widely used.
<P>Kazuhiko Miki rewrote my LZSS in Turbo Pascal and assembly language, and soon made it evolve into a
complete archiver, which he named LARC.
<P>The first versions of LZSS and LARC were rather slow. So I rewrote my LZSS using a binary tree, and so
did Miki. Although LARC's encoding was slower than the fastest archiver available, its decoding was
quite fast, and its algorithm was so simple that even self-extracting files (compressed files plus
decoder) it created were usually smaller than non-self-extracting files from other archivers.
<P>Soon many hobby programmers joined the archiver project at the forum. Very many suggestions were made,
and LARC was revised again and again. By the summer of 1988, LARC's speed and compression have improved
so much that LARC-compressed programs were beginning to be uploaded in many forums of PC-VAN and other
networks.
<P>In that summer I wrote another program, LZARI, which combined the LZSS algorithm with adaptive arithmetic
compression. Although it was slower than LZSS, its compression performance was amazing.
<P>Miki, the author of LARC, uploaded LZARI to NIFTY-Serve, another big information network in Japan. In
NIFTY-Serve, Haruyasu Yoshizaki replaced LZARI's adaptive arithmetic coding with a version of adaptive
Huffman coding to increase speed. Based on this algorithm, which he called LZHUF, he developed yet
another archiver, LHarc.
<P>In what follows, I will review several of these algorithms and supply simplified codes in C language.
<H1>Simple coding methods</H1>
<P>Replacing several (usually 8 or 4) "space" characters by one "tab" character is a very primitive method
for data compression. Another simple method is run-length coding, which encodes the message
"AAABBBBAACCCC" into "3A4B2A4C", for example.
<H1>LZSS coding</H1>
<P>This scheme is initiated by Ziv and Lempel [<A HREF="#ref">1</A>]. A slightly modified version is
described by Storer and Szymanski [<A HREF="#ref">2</A>]. An implementation using a binary tree is
proposed by Bell [<A HREF="#ref">3</A>]. The algorithm is quite simple: Keep a ring buffer, which
initially contains "space" characters only. Read several letters from the file to the buffer. Then
search the buffer for the longest string that matches the letters just read, and send its length and
position in the buffer.
<P>If the buffer size is 4096 bytes, the position can be encoded in 12 bits. If we represent the match
length in four bits, the [position, length] pair is two bytes long. If the longest match is no more than
two characters, then we send just one character without encoding, and restart the process with the next
letter. We must send one extra bit each time to tell the decoder whether we are sending a [position,
length] pair or an unencoded character.
<P>The accompanying file LZSS.C is a version of this algorithm. This implementation uses multiple binary
trees to speed up the search for the longest match. All the programs in this article are written in
draft-proposed ANSI C. I tested them with Turbo C 2.0.
<H1>LZW coding</H1>
<P>This scheme was devised by Ziv and Lempel [<A HREF="#ref">4</A>], and modified by Welch [<A
HREF="#ref">5</A>].
<P>The LZW coding has been adopted by most of the existing archivers, such as ARC and PKZIP. The algorithm
can be made relatively fast, and is suitable for hardware implementation as well.
<P>The algorithm can be outlined as follows: Prepare a table that can contain several thousand items.
Initially register in its 0th through 255th positions the usual 256 characters. Read several letters
from the file to be encoded, and search the table for the longest match. Suppose the longest match is
given by the string "ABC". Send the position of "ABC" in the table. Read the next character from the
file. If it is "D", then register a new string "ABCD" in the table, and restart the process with the
letter "D". If the table becomes full, discard the oldest item or, preferably, the least used.
<P>A Pascal program for this algorithm is given in Storer's book [<A HREF="#ref">6</A>].
<H1>Huffman coding</H1>
<P>Classical Huffman coding is invented by Huffman [<A HREF="#ref">7</A>]. A fairly readable account is
given in Sedgewick [<A HREF="#ref">8</A>].
<P>Suppose the text to be encoded is "ABABACA", with four A's, two B's, and a C. We represent this situation
as follows:
<P>
<CENTER>
<TABLE>
<TR>
<TD><PRE><DIV CLASS="code">4 2 1
| | |
A B C
</DIV></PRE>
</TD>
</TR>
</TABLE>
</CENTER>
<P>Combine the least frequent two characters into one, resulting in the new frequency 2 + 1 = 3:
<P>
<CENTER>
<TABLE>
<TR>
<TD><PRE><DIV CLASS="code">4 3
| / \
A B C
</DIV></PRE>
</TD>
</TR>
</TABLE>
</CENTER>
<P>Repeat the above step until the whole characters combine into a tree:
<P>
<CENTER>
<TABLE>
<TR>
<TD><PRE><DIV CLASS="code"> 7
/ \
/ 3
/ / \
A B C
</DIV></PRE>
</TD>
</TR>
</TABLE>
</CENTER>
<P>Start at the top ("root") of this encoding tree, and travel to the character you want to encode. If you
go left, send a "0"; otherwise send a "1". Thus, "A" is encoded by "0", "B" by "10", "C" by "11".
Algotether, "ABABACA" will be encoded into ten bits, "0100100110".
<P>To decode this code, the decoder must know the encoding tree, which must be sent separately.
<P>A modification to this classical Huffman coding is the adaptive, or dynamic, Huffman coding. See, e.g.,
Gallager [<A HREF="#ref">9</A>]. In this method, the encoder and the decoder processes the first letter
of the text as if the frequency of each character in the file were one, say. After the first letter has
been processed, both parties increment the frequency of that character by one. For example, if the first
letter is 'C', then freq['C'] becomes two, whereas every other frequencies are still one. Then the both
parties modify the encoding tree accordingly. Then the second letter will be encoded and decoded, and so
on.
<H1>Arithmetic coding</H1>
<P>The original concept of arithmetic coding is proposed by P. Elias. An implementation in C language is
described by Witten and others [<A HREF="#ref">10</A>].
<P>Although the Huffman coding is optimal if each character must be encoded into a fixed (integer) number of
bits, arithmetic coding wins if no such restriction is made.
<P>As an example we shall encode "AABA" using arithmetic coding. For simplicity suppose we know beforehand
that the probabilities for "A" and "B" to appear in the text are 3/4 and 1/4, respectively.
<P>Initially, consider an interval:
<P>
<CENTER><PRE><DIV CLASS="code">0 <= x < 1.
</DIV></PRE>
</CENTER>
<P>Since the first character is "A" whose probability is 3/4, we shrink the interval to the lower 3/4:
<P>
<CENTER><PRE><DIV CLASS="code">0 <= x < 3/4.
</DIV></PRE>
</CENTER>
<P>The next character is "A" again, so we take the lower 3/4:
<P>
<CENTER><PRE><DIV CLASS="code">0 <= x < 9/16.
</DIV></PRE>
</CENTER>
<P>Next comes "B" whose probability is 1/4, so we take the upper 1/4:
<P>
<CENTER><PRE><DIV CLASS="code">27/64 <= x < 9/16,
</DIV></PRE>
</CENTER>
<P>because "B" is the second element in our alphabet, {A, B}. The last character is "A" and the interval is
<P>
<CENTER><PRE><DIV CLASS="code">27/64 <= x < 135/256,
</DIV></PRE>
</CENTER>
<P>which can be written in binary notation
<P>
<CENTER><PRE><DIV CLASS="code">0.011011 <= x < 0.10000111.
</DIV></PRE>
</CENTER>
<P>Choose from this interval any number that can be represented in fewest bits, say 0.1, and send the bits
to the right of "0."; in this case we send only one bit, "1". Thus we have encoded four letters into one
bit! With the Huffman coding, four letters could not be encoded into less than four bits.
<P>To decode the code "1", we just reverse the process: First, we supply the "0." to the right of the
received code "1", resulting in "0.1" in binary notation, or 1/2. Since this number is in the first 3/4
of the initial interval 0 <= x < 1, the first character must be "A". Shrink the interval into the lower
3/4. In this new interval, the number 1/2 lies in the lower 3/4 part, so the second character is again
"A", and so on. The number of letters in the original file must be sent separately (or a special 'EOF'
character must be appended at the end of the file).
<P>The algorithm described above requires that both the sender and receiver know the probability
distribution for the characters. The adaptive version of the algorithm removes this restriction by first
supposing uniform or any agreed-upon distribution of characters that approximates the true distribution,
and then updating the distribution after each character is sent and received.
<H1>LZARI</H1>
<P>In each step the LZSS algorithm sends either a character or a [position, length] pair. Among these,
perhaps character "e" appears more frequently than "x", and a [position, length] pair of length 3 might
be commoner than one of length 18, say. Thus, if we encode the more frequent in fewer bits and the less
frequent in more bits, the total length of the encoded text will be diminished. This consideration
suggests that we use Huffman or arithmetic coding, preferably of adaptive kind, along with LZSS.
<P>This is easier said than done, because there are many possible [position, length] combinations. Adaptive
compression must keep running statistics of frequency distribution. Too many items make statistics
unreliable.
<P>What follows is not even an approximate solution to the problem posed above, but anyway this was what I
did in the summer of 1988.
<P>I extended the character set from 256 to three-hundred or so in size, and let characters 0 through 255 be
the usual 8-bit characters, whereas characters 253 + n represent that what follows is a position of
string of length n, where n = 3, 4 , .... These extended set of characters will be encoded with adaptive
arithmetic compression.
<P>I also observed that longest-match strings tend to be the ones that were read relatively recently.
Therefore, recent positions should be encoded into fewer bits. Since 4096 positions are too many to
encode adaptively, I fixed the probability distribution of the positions "by hand." The distribution
function given in the accompanying LZARI.C is rather tentative; it is not based on thorough
experimentation. In retrospect, I could encode adaptively the most significant 6 bits, say, or perhaps
by some more ingenious method adapt the parameters of the distribution function to the running
statistics.
<P>At any rate, the present version of LZARI treats the positions rather separately, so that the overall
compression is by no means optimal. Furthermore, the string length threshold above which strings are
coded into [position, length] pairs is fixed, but logically its value must change according to the
length of the [position, length] pair we would get.
<H1>LZHUF</H1>
<P>LZHUF, the algorithm of Haruyasu Yoshizaki's archiver LHarc, replaces LZARI's adaptive arithmetic coding
with adaptive Huffman. LZHUF encodes the most significant 6 bits of the position in its 4096-byte buffer
by table lookup. More recent, and hence more probable, positions are coded in less bits. On the other
hand, the remaining 6 bits are sent verbatim. Because Huffman coding encodes each letter into a fixed
number of bits, table lookup can be easily implemented.
<P>Though theoretically Huffman cannot exceed arithmetic compression, the difference is very slight, and
LZHUF is fairly fast.
<P>The accompanying file LZHUF.C was written by Yoshizaki. I translated the comments into English and made a
few trivial changes to make it conform to the ANSI C standard.
<A NAME="ref"><H1>References</H1></A>
<DL COMPACT>
<DT>[1]
<DD>J. Ziv and A. Lempel, IEEE Trans. IT-23, 337-343 (1977).
<DT>[2]
<DD>J. A. Storer and T. G. Szymanski, J. ACM, 29, 928-951 (1982).
<DT>[3]
<DD>T. C. Bell, IEEE Trans. COM-34, 1176-1182 (1986).
<DT>[4]
<DD>J. Ziv and A. Lempel, IEEE Trans. IT-24, 530-536 (1978).
<DT>[5]
<DD>T. A. Welch, Computer, 17, No.6, 8-19 (1984).
<DT>[6]
<DD>J. A. Storer, Data Compression: Methods and Theory (Computer Science Press, 1988).
<DT>[7]
<DD>D. A. Huffman, Proc IRE 40, 1098-1101 (1952).
<DT>[8]
<DD>R. Sedgewick, Algorithms, 2nd ed. (Addison-Wesley, 1988).
<DT>[9]
<DD>R. G. Gallager, IEEE Trans. IT-24, 668-674 (1978).
<DT>[10]
<DD>I. E. Witten, R. M. Neal, and J. G. Cleary, Commun. ACM 30, 520-540 (1987).
<p align="center"><b><a
href="http://archive.gamedev.net/community/forums/topic.asp?key=featart&amp;uid=295&amp;forum_id=35&amp;Topic_Title=Data+Compression+Algorithms+of+LARC+and+LHarc">Discuss
this article in the forums</a></b></p>
<p>
<br><span class="maintext-2">Date this article was posted to GameDev.net: <b>7/16/1999</b>
<br>(Note that this date does not necessarily correspond to the date the article was written)</span>
<p><b>See Also:</b><br>
<a href="../list6d5b.html?categoryid=125">Compression Algorithms </a><br>
<p align="center">&copy; 1999-2011 Gamedev.net. All rights reserved. <a
href="../../info/legal.htm#copyright"><u>Terms of Use</u></a> <a
href="../../info/legal.htm#privacy"><u>Privacy Policy</u></a>
<br><span class="maintext-1">Comments? Questions? Feedback? <a href="../../info/faq.html">Click here!</a></span>
</p>
</td>
</tr>
</table>
<!-- start Vibrant Media IntelliTXT Tooltip style sheet -->
<style type="text/css">
.iTt {
FONT-FAMILY: Verdana, Arial, Helvetica;
FONT-SIZE: 11px;
FONT-STYLE: normal;
FONT-WEIGHT: normal;
COLOR: black;
BACKGROUND-COLOR: lightyellow;
BORDER: black 1px solid;
PADDING: 2px;
}
.iTt a {
COLOR: 0000CC;
}
.iTt a:visited {
COLOR: 0000CC;
}
.iTt a:hover {
COLOR: 6666CC;
}
.iTt TD {
COLOR: 999999;
}
</style>
<!-- end Vibrant Media IntelliTXT style sheet -->
<!-- start Vibrant Media IntelliTXT script section -->
<script src="http://gamedev.us.intellitxt.com/intellitxt/front.asp?ipid=1966" type="text/javascript"></script>
<!-- end Vibrant Media IntelliTXT script section -->
</body>
</html>
fxl.codes.kisekae.data/Docs/LZH format.html
+4222
View File
File diff suppressed because it is too large Load Diff
fxl.codes.kisekae/Docs/fkiss.txt → fxl.codes.kisekae.data/Docs/fkiss.txt
View File
fxl.codes.kisekae/Docs/kissfrmt.txt → fxl.codes.kisekae.data/Docs/kissfrmt.txt
View File
fxl.codes.kisekae.sln
+6
View File
@@ -6,6 +6,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "fxl.codes.kisekae.blazor",
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "fxl.codes.kisekae.data", "fxl.codes.kisekae.data\fxl.codes.kisekae.data.csproj", "{A977AF77-5A7F-4DDA-8113-3367EE9920BB}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "fxl.codes.kisekae.data", "fxl.codes.kisekae.data\fxl.codes.kisekae.data.csproj", "{A977AF77-5A7F-4DDA-8113-3367EE9920BB}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "fxl.codes.kisekae.data.test", "fxl.codes.kisekae.data.test\fxl.codes.kisekae.data.test.csproj", "{1D7F531D-38FD-4C0C-B326-1AFF697E5882}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@@ -24,5 +26,9 @@ Global
{A977AF77-5A7F-4DDA-8113-3367EE9920BB}.Debug|Any CPU.Build.0 = Debug|Any CPU {A977AF77-5A7F-4DDA-8113-3367EE9920BB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A977AF77-5A7F-4DDA-8113-3367EE9920BB}.Release|Any CPU.ActiveCfg = Release|Any CPU {A977AF77-5A7F-4DDA-8113-3367EE9920BB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A977AF77-5A7F-4DDA-8113-3367EE9920BB}.Release|Any CPU.Build.0 = Release|Any CPU {A977AF77-5A7F-4DDA-8113-3367EE9920BB}.Release|Any CPU.Build.0 = Release|Any CPU
{1D7F531D-38FD-4C0C-B326-1AFF697E5882}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1D7F531D-38FD-4C0C-B326-1AFF697E5882}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1D7F531D-38FD-4C0C-B326-1AFF697E5882}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1D7F531D-38FD-4C0C-B326-1AFF697E5882}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal