Добавлена первая версия ЛР по сжатию изображений при помощи нейронных сетей.

Версия сделана для учебных целей.

EntropyCompile.bat

+del EntropyCodec.cp39-win_amd64.pyd
+del ..\EntropyCodec.cp39-win_amd64.pyd
+python.exe EntropySetup.py build_ext --inplace
+pause
\ No newline at end of file

EntropySetup.py

+from setuptools import setup, Extension
+import os
+import pybind11
+
+functions_module = Extension(
+    name='EntropyCodec',
+    sources=['wrapper.cpp'],
+    include_dirs=[os.path.join(os.getenv('PYTHON_DIR'), 'include'),
+                os.path.join(pybind11.__path__[0], 'include')]
+)
+
+setup(ext_modules=[functions_module], options={"build_ext": {"build_lib": ".."}})

ImageCoder.cpp

+// ImageCodec.cpp : This file contains the 'main' function. Program execution begins and ends there.
+//
+#define _CRT_SECURE_NO_WARNINGS
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+
+#include "mcoder.h"
+
+int BitPlaneEncoder(unsigned char *obuffer, unsigned char *layer1, int w1, int h1, int z1, int *bitsize)
+{
+	BiContextType bin_ctx;
+	int code_len = 0;
+	EncodingEnvironmentPtr eep;
+	unsigned char *bufptr = obuffer;
+
+	eep = arienco_create_encoding_environment();
+	arienco_start_encoding(eep, bufptr, &code_len);
+	biari_init_context(&bin_ctx, "ctx");
+
+	for (int z = 0;z < z1;z++)
+	{
+		for (int i = 0;i < h1;i++)
+		{
+			for (int j = 0;j < w1;j++)
+			{
+				biari_encode_symbol(eep, layer1[z*h1*w1+i*w1+j], &bin_ctx);
+			}
+		}
+	}
+
+	biari_encode_symbol(eep, 0, &bin_ctx);
+	biari_encode_symbol(eep, 1, &bin_ctx);
+	biari_encode_symbol(eep, 0, &bin_ctx);
+	biari_encode_symbol(eep, 1, &bin_ctx);
+
+	arienco_done_encoding(eep);
+	arienco_delete_encoding_environment(eep);
+	*bitsize = code_len / 8;
+
+	return code_len / 8;
+}
+
+int BitPlaneDecoder(unsigned char *obuffer, unsigned char *layer1, int w1, int h1, int z1, int *offset)
+{
+	BiContextType bin_ctx;
+	int code_len = 0;
+	DecodingEnvironmentPtr dep;
+	unsigned char *bufptr = obuffer;
+
+	bufptr += (*offset);
+	dep = arideco_create_decoding_environment();
+	arideco_start_decoding(dep, bufptr, 0, &code_len);
+	biari_init_context(&bin_ctx, "ctx");
+
+	for (int z = 0;z < z1;z++)
+	{
+		for (int i = 0;i < h1;i++)
+		{
+			for (int j = 0;j < w1;j++)
+			{
+				layer1[z*h1*w1 + i * w1 + j] = biari_decode_symbol(dep, &bin_ctx);
+			}
+		}
+	}
+	*offset = code_len;
+	arideco_delete_decoding_environment(dep);
+
+	return 0;
+}
+
+int main(int argc, char* argv[])
+{
+	
+	return 0;
+}
\ No newline at end of file

ac_dec.cpp

+#include <stdlib.h>
+#include <stdio.h>
+#include "mcoder.h"
+#include <math.h>
+
+
+/*!
+ ************************************************************************
+ * \brief
+ *    Allocates memory for the DecodingEnvironment struct
+ * \return DecodingContextPtr
+ *    allocates memory
+ ************************************************************************
+ */
+DecodingEnvironmentPtr arideco_create_decoding_environment()
+{
+  DecodingEnvironmentPtr dep;
+
+  if ((dep = (DecodingEnvironmentPtr)calloc(1,sizeof(DecodingEnvironment))) == NULL)
+    printf("arideco_create_decoding_environment: dep");
+  return dep;
+}
+
+
+/*!
+ ***********************************************************************
+ * \brief
+ *    Frees memory of the DecodingEnvironment struct
+ ***********************************************************************
+ */
+void arideco_delete_decoding_environment(DecodingEnvironmentPtr dep)
+{
+  free(dep);
+}
+
+
+/*!
+ ************************************************************************
+ * \brief
+ *    Initializes the DecodingEnvironment for the arithmetic coder
+ ************************************************************************
+ */
+void arideco_start_decoding(DecodingEnvironmentPtr dep, unsigned char *cpixcode,
+                            int firstbyte, int *cpixcode_len )
+{
+  register unsigned int bbit = 0;
+ 
+  int value = 0;
+  dep->Dcodestrm = cpixcode;
+  dep->Dcodestrm_len = (unsigned int*)cpixcode_len;
+  dep->Dvalue = 0;
+  for (int i = 1; i <= BITS_IN_REGISTER; i++)
+  {
+	Get1Bit(dep->Dcodestrm, *dep->Dcodestrm_len, bbit);
+    dep->Dvalue = 2 * dep->Dvalue + bbit;
+  }
+  dep->Dlow = 0;
+  dep->Drange = HALF-2;
+}
+
+unsigned int  biari_decode_symbol(DecodingEnvironmentPtr dep, BiContextTypePtr bi_ct)
+{
+  int bbit;
+  register unsigned int low = dep->Dlow;
+  register unsigned int value = dep->Dvalue;
+  register unsigned int range = dep->Drange;
+  unsigned int symbol;
+
+  unsigned int cum = (int)((((long) (value - low) + 1) * bi_ct->freq_all - 1) / range);
+  for (int i=0;i<ALPHABET_SIZE;i++)
+  {
+	if (bi_ct->cum_freq [i] > cum)
+	{
+		break;		
+	}
+	symbol = i;
+  }
+
+  low = low + (range * bi_ct->cum_freq [symbol]) / bi_ct->freq_all;
+  range = (range*bi_ct->freq [symbol])/bi_ct->freq_all;
+  range = max(1,range);
+
+  bi_ct->freq[symbol]++;
+  bi_ct->freq_all++;
+  if (bi_ct->freq_all>16384)
+  {
+	bi_ct->freq_all=0;
+	for (int i=0;i<ALPHABET_SIZE;i++)
+	{
+		bi_ct->freq[i]=max(1,bi_ct->freq[i]>>1);
+		bi_ct->freq_all+=bi_ct->freq[i];
+	}
+  }
+  bi_ct->cum_freq[0]=0;
+  for (int i=1;i<=ALPHABET_SIZE;i++)
+  {
+	bi_ct->cum_freq[i] = bi_ct->cum_freq[i-1]+bi_ct->freq[i-1];
+  }
+
+  while (range < QUARTER)
+  {
+ 	if (low >= HALF)
+    {
+        low -= HALF;
+		value -= HALF;
+    }
+    else 
+      if (low < QUARTER)
+      {
+      }
+      else
+      {
+        low -= QUARTER;
+		value -= QUARTER;
+      }
+    // Double range 
+    range <<= 1;
+	low <<= 1;
+	Get1Bit(dep->Dcodestrm, *dep->Dcodestrm_len, bbit);
+    value = (value << 1) | bbit;
+  }
+
+  dep->Drange = range;
+  dep->Dvalue = value;
+  dep->Dlow = low;
+ 
+  return(symbol);
+}
+
+

ac_enc.cpp

+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+#include "mcoder.h"
+
+/*!
+ ************************************************************************
+ * \brief
+ *    Initializes a given context with some pre-defined probability state
+ ************************************************************************
+ */
+void biari_init_context 
+(
+	BiContextTypePtr ctx, 
+	char* name
+)
+{
+  ctx->freq_all = 0;
+  for (int i=0;i<ALPHABET_SIZE;i++)
+  {
+	ctx->freq[i] = 1;
+	ctx->freq_all+=ctx->freq[i];
+  }
+
+  ctx->cum_freq[0]=0;
+  for (int i=1;i<=ALPHABET_SIZE;i++)
+  {
+	  ctx->cum_freq[i] = ctx->cum_freq[i-1]+ctx->freq[i-1];
+  }
+}
+
+/*!
+ ************************************************************************
+ * \brief
+ *    Allocates memory for the EncodingEnvironment struct
+ ************************************************************************
+ */
+EncodingEnvironmentPtr arienco_create_encoding_environment()
+{
+  EncodingEnvironmentPtr eep;
+
+  if ( (eep = (EncodingEnvironmentPtr) calloc(1,sizeof(EncodingEnvironment))) == NULL)
+    printf("arienco_create_encoding_environment: eep");
+
+  return eep;
+}
+
+/*!
+ ************************************************************************
+ * \brief
+ *    Frees memory of the EncodingEnvironment struct
+ ************************************************************************
+ */
+void arienco_delete_encoding_environment(EncodingEnvironmentPtr eep)
+{
+  if (eep != NULL)
+  {
+    free(eep);
+  }
+}
+
+/*!
+ ************************************************************************
+ * \brief
+ *    Initializes the EncodingEnvironment for the arithmetic coder
+ ************************************************************************
+ */
+void arienco_start_encoding(EncodingEnvironmentPtr eep,
+                            unsigned char *code_buffer,
+                            int *code_len )
+{
+  eep->Elow = 0;
+  eep->Erange = HALF-2;
+  eep->Ebits_to_follow = 0;
+  eep->Ecodestrm = code_buffer;
+  eep->Ecodestrm_len = (unsigned int*)code_len;
+}
+
+/*!
+ ************************************************************************
+ * \brief
+ *    Terminates the arithmetic codeword, writes stop bit and stuffing bytes (if any)
+ ************************************************************************
+ */
+
+void arienco_done_encoding(EncodingEnvironmentPtr eep)
+{
+  if((eep->Elow >> (BITS_IN_REGISTER-1)) & 1)
+  {
+    put_one_bit_1_plus_outstanding;
+  }
+  else
+  {
+    put_one_bit_0_plus_outstanding;
+  }
+  
+  Put1Bit(eep->Ecodestrm, *eep->Ecodestrm_len, (eep->Elow >> (BITS_IN_REGISTER-2))&1);
+  Put1Bit(eep->Ecodestrm, *eep->Ecodestrm_len, 1);
+    *eep->Ecodestrm_len = (*eep->Ecodestrm_len+7) & ~7;
+}
+
+
+
+void biari_encode_symbol(EncodingEnvironmentPtr eep, signed int symbol, BiContextTypePtr bi_ct)
+{
+  register unsigned int range = eep->Erange;
+  register unsigned int low = eep->Elow;
+
+  low = low + (range * bi_ct->cum_freq [symbol]) / bi_ct->freq_all;
+  range = (range*bi_ct->freq [symbol])/bi_ct->freq_all;
+  range=max(1,range);
+
+  bi_ct->freq[symbol]++;
+  bi_ct->freq_all++;
+  if (bi_ct->freq_all>16384)
+  {
+	bi_ct->freq_all=0;
+	for (int i=0;i<ALPHABET_SIZE;i++)
+	{
+		bi_ct->freq[i]=max(1,bi_ct->freq[i]>>1);
+		bi_ct->freq_all+=bi_ct->freq[i];
+	}
+  }
+  bi_ct->cum_freq[0]=0;
+  for (int i=1;i<=ALPHABET_SIZE;i++)
+  {
+	bi_ct->cum_freq[i] = bi_ct->cum_freq[i-1]+bi_ct->freq[i-1];
+  }
+
+  // renormalization 
+  while (range < QUARTER)
+  {
+    if (low >= HALF)
+    {
+      put_one_bit_1_plus_outstanding;
+      low -= HALF;
+    }
+    else 
+      if (low < QUARTER)
+      {
+        put_one_bit_0_plus_outstanding;
+      }
+      else
+      {
+        eep->Ebits_to_follow++;
+        low -= QUARTER;
+      }
+    low <<= 1;
+    range <<= 1;
+  }
+  
+  eep->Erange = range;
+  eep->Elow = low;
+  
+  bi_ct->freq_stat[symbol]++;
+}
+

decoder.mdl/fingerprint.pb

+ގœ浂
* ȾG(e2

\ No newline at end of file

encoder.mdl/fingerprint.pb

+
Ֆ֯
i ǹL(Ͳ
2

\ No newline at end of file

mcoder.h

+#ifndef MCODER_H
+#define MCODER_H
+
+#include "stream.h"
+#include <stdio.h>
+
+//new parameters
+
+#define BITS_IN_REGISTER 16
+#define TOP_VALUE (((long) 1 << BITS_IN_REGISTER) - 1)
+#define QUARTER (TOP_VALUE / 4 + 1)
+#define FIRST_QTR (TOP_VALUE / 4 + 1)
+#define HALF      (2 * FIRST_QTR)
+#define THIRD_QTR (3 * FIRST_QTR)
+
+#define ALPHABET_SIZE 256
+#define max(a,b) (((a) > (b)) ? (a) : (b))
+
+typedef struct
+{
+  unsigned long freq[ALPHABET_SIZE+1];
+  unsigned long freq_all;
+  unsigned long cum_freq[ALPHABET_SIZE+1];
+  unsigned long freq_stat[ALPHABET_SIZE+1];
+} BiContextType;
+
+typedef BiContextType *BiContextTypePtr;
+
+#define putLogToFile(fname,...) { \
+  FILE *fp = fopen(fname,"at"); \
+  fprintf(fp,__VA_ARGS__); \
+  fclose(fp); \
+}
+
+
+
+#define SIGN_TO_ENCODE(x) ((x) >= 0 ? 0 : 1)
+#define CABAC_ABS(x) ((x) > 0 ? (x) : -(x))
+#define MAX_BITS_IN_SERIE 25
+///   25 MAXIMUM
+  #define PutZeros(nbits) {\
+    unsigned int iii;\
+    for(iii = 0; iii<(nbits); iii++ )\
+    {\
+      Put1Bit(eep->Ecodestrm, *eep->Ecodestrm_len, 0);\
+    }\
+      }
+
+  #define put_one_bit_1_plus_outstanding { \
+                                          Put1Bit(eep->Ecodestrm, *eep->Ecodestrm_len, 1); \
+                                          PutZeros(eep->Ebits_to_follow);\
+                                          eep->Ebits_to_follow = 0;\
+                                         }
+
+  #define PutLongOnes(nbits) {\
+      unsigned int i1=0xFFFFFFFF;\
+      int bits1;\
+      int main = (nbits)/MAX_BITS_IN_SERIE;\
+      int tail = (nbits)%MAX_BITS_IN_SERIE;\
+      for(bits1 = 0; bits1<main; bits1++ ) {\
+        PutBits(eep->Ecodestrm, *eep->Ecodestrm_len,i1,MAX_BITS_IN_SERIE);\
+      }\
+      PutBits(eep->Ecodestrm, *eep->Ecodestrm_len,i1,tail);\
+    }
+
+  #define put_one_bit_0_plus_outstanding { \
+                                          Put1Bit(eep->Ecodestrm, *eep->Ecodestrm_len, 0); \
+                                          if( eep->Ebits_to_follow > MAX_BITS_IN_SERIE ) \
+                                          { \
+                                            PutLongOnes(eep->Ebits_to_follow);\
+                                          }\
+                                          else \
+                                          {\
+                                            PutBitsOnes(eep->Ecodestrm, *eep->Ecodestrm_len, eep->Ebits_to_follow);\
+                                          }\
+                                          eep->Ebits_to_follow = 0;\
+                                         }
+
+
+//! struct to characterize the state of the arithmetic coding engine
+typedef struct
+{
+  unsigned int  Elow, Erange, Ehigh;
+  unsigned int  Ebits_to_follow;
+  unsigned char *Ecodestrm;
+  unsigned int  *Ecodestrm_len;
+} EncodingEnvironment;
+
+typedef EncodingEnvironment *EncodingEnvironmentPtr;
+
+// cabac_enc.c
+void biari_init_context 
+(
+	BiContextTypePtr ctx, 
+	char* name
+);
+
+
+EncodingEnvironmentPtr arienco_create_encoding_environment();
+void arienco_delete_encoding_environment(EncodingEnvironmentPtr eep);
+void arienco_start_encoding(EncodingEnvironmentPtr eep,
+                            unsigned char *code_buffer,
+                            int *code_len );
+void arienco_done_encoding(EncodingEnvironmentPtr eep);
+
+void biari_encode_symbol(EncodingEnvironmentPtr eep, int symbol, BiContextTypePtr bi_ct);
+
+
+//! struct to characterize the state of the arithmetic coding engine
+typedef struct
+{
+  unsigned int    Dlow, Drange, Dhigh;
+  unsigned int    Dvalue;
+  unsigned int    Dbuffer;
+  int             Dbits_to_go;
+  unsigned char   *Dcodestrm;
+  unsigned int    *Dcodestrm_len;
+} DecodingEnvironment;
+
+typedef DecodingEnvironment *DecodingEnvironmentPtr;
+
+// cabac_dec.c
+DecodingEnvironmentPtr arideco_create_decoding_environment();
+void arideco_delete_decoding_environment(DecodingEnvironmentPtr dep);
+void arideco_start_decoding(DecodingEnvironmentPtr dep, unsigned char *cpixcode,
+                            int firstbyte, int *cpixcode_len );
+int arideco_bits_read(DecodingEnvironmentPtr dep);
+void arideco_done_decoding(DecodingEnvironmentPtr dep);
+unsigned int biari_decode_symbol(DecodingEnvironmentPtr dep, BiContextTypePtr bi_ct);
+#endif // !MCODER_H
\ No newline at end of file

stream.h

+#ifndef _STREAM_H
+#define _STREAM_H
+
+// ****************************************************
+// *     Bits Order (w/o swap):                       *
+// * the 1st bit in a bitstream is 0th bit (not 7th)  *
+// ****************************************************
+
+#define mask(y,nb) ((unsigned int)((int)(y) & ((1L<<(nb)) - 1)))
+
+#ifdef TMS
+#define UINT32P(x)  _mem4(x)
+#else
+#define UINT32P(x)  (*((unsigned int*)(x)))
+#endif
+
+#define UINT32(x) ((unsigned int)(x))
+#define UCHAR(x)  ((unsigned char)(x))
+#define UCHARP(x) ((unsigned char*)(x))
+
+//-----------------------------------------------
+#define PutBitsAC(stream, bitptr, x, nbits)   	\
+{ 												\
+   UINT32P(&(UCHARP(stream)[(bitptr)>>3]))  |=	\
+           UINT32(x) << ((bitptr)&0x7);         \
+   bitptr += nbits;                             \
+}												\
+//-----------------------------------------------
+//-------------------------------------------------------------------
+#define Put1BitAC(stream, bitptr, x)  								\
+{       				 	  										\
+  if(x) UCHARP(stream)[(bitptr)>>3] |= UCHAR(1L<<((bitptr)&0x7));	\
+  (bitptr)++;                                                	  	\
+}																	\
+//-------------------------------------------------------------------
+
+//---------------------------------------------------
+#define PutBits(stream, bitptr, x, nbits)    		\
+{                                         			\
+   UINT32P(&(UCHARP(stream)[((bitptr)+7)>>3])) = 0;	\
+   UINT32P(&(UCHARP(stream)[(bitptr)>>3]))    |= 	\
+           UINT32(mask(x,nbits) << ((bitptr)&0x7));	\
+   bitptr += nbits;									\
+}													\
+//---------------------------------------------------
+
+//---------------------------------------------------
+#define PutBitsOnes(stream, bitptr, nbits)       	\
+{                                                   \
+   UINT32P(&(UCHARP(stream)[((bitptr)+7)>>3])) = 0;	\
+   UINT32P(&(UCHARP(stream)[(bitptr)>>3]))    |=    \
+    UINT32(mask(0xFFFFFFFF,nbits) << ((bitptr)&0x7)); \
+   bitptr += nbits;                                 \
+}													\
+//---------------------------------------------------
+//---------------------------------------------------
+#define PutBitsZeros(stream, bitptr, nbits)     	\
+{                                                   \
+   UINT32P(&(UCHARP(stream)[((bitptr)+7)>>3])) = 0;	\
+   bitptr += nbits;                                 \
+}													\
+//---------------------------------------------------
+
+//------------------------------------------------------------------
+#define ShiftBitPtr(bitptr, nbits) bitptr += nbits;
+//------------------------------------------------------------------
+
+// *********************************************
+// *    Put a bit into the bitstream.          *
+// * x can be any value;                       *
+// * only if x~=0, the current bit is set to 1 *
+// *********************************************
+//---------------------------------------------------------------
+#define Put1Bit(stream, bitptr, x)                  			\
+{                                                       		\
+   UINT32P(&(UCHARP(stream)[((bitptr)+7)>>3]) ) = 0;     		\
+   if((x)!=0) UINT32P(&(UCHARP(stream)[(bitptr)>>3])) |=       	\
+                                UINT32(1L << ((bitptr)&0x7));	\
+   (bitptr)++;                                            		\
+}																\
+//---------------------------------------------------------------
+
+//-------------------------------------------------------
+#define Put1BitOne(stream, bitptr)                   	\
+{                                                       \
+   UINT32P(&(UCHARP(stream)[((bitptr)+7)>>3]) ) = 0;	\
+   UINT32P(&(UCHARP(stream)[(bitptr)>>3]) )    |=  		\
+               	       UINT32(1L << ((bitptr)&0x7));	\
+   (bitptr)++;                                          \
+}														\
+//-------------------------------------------------------
+//-------------------------------------------------------
+#define Put1BitZero(stream, bitptr)                   	\
+{                                                       \
+   UINT32P(&(UCHARP(stream)[((bitptr)+7)>>3]) ) = 0; 	\
+   (bitptr)++;                                          \
+}														\
+//-------------------------------------------------------
+
+//-------------------------------------------------------
+#define Get1BitCABAC(stream, bitptr, x)                                 \
+{                                                                  \
+   x   = (stream)[(bitptr)>>3];   \
+   x >>= (bitptr) & 0x7 ;                                          \
+   x  &= 1;                                                        \
+   (bitptr)++;                                                     \
+}
+
+//-------------------------------------------------------
+#define GetBits(stream, bitptr, x, nbits)                      \
+{                                                       \
+   x   = UINT32P(&(UCHARP(stream)[(bitptr)>>3]));	\
+   x >>= (bitptr) & 0x7 ;                               \
+   bitptr += nbits;                                          \
+}														\
+//-------------------------------------------------------
+
+
+//-------------------------------------------------------
+#define Get1Bit(stream, bitptr, x)                      \
+{                                                       \
+   x   = UINT32P(&(UCHARP(stream)[(bitptr)>>3]));	\
+   x >>= (bitptr) & 0x7 ;                               \
+   x  &= 1;                                             \
+   (bitptr)++;                                          \
+}														\
+//-------------------------------------------------------
+#define Get1BitCABAC(stream, bitptr, x)                                 \
+{                                                                  \
+   x   = (stream)[(bitptr)>>3];   \
+   x >>= (bitptr) & 0x7 ;                                          \
+   x  &= 1;                                                        \
+   (bitptr)++;                                                     \
+}
+
+#endif // _STREAM_H