Bitcoin Forum

int dw_zip_block = 0;
int dw_zip_limit_size = 0;
int dw_zip_txdb = 0;

bool AppInit2()
{
...
    // ********************************************************* Step 2: parameter interactions

#ifdef WIN32
    dw_zip_block = GetArg("-zipblock", 1);
#else
    /* 7Zip source code in the Linux system needs to improve, It can work, but sometimes it will crash. */
    dw_zip_block = GetArg("-zipblock", 0);
#endif
    dw_zip_limit_size = GetArg("-ziplimitsize", 64);
    dw_zip_txdb = GetArg("-ziptxdb", 0);
    if( dw_zip_block > 1 ){ dw_zip_block = 1; }
    else if( dw_zip_block == 0 ){ dw_zip_txdb = 0; }

...
}
 

extern int bitnet_pack_block(CBlock* block, string& sRzt);
extern bool getCBlockByFilePos(CAutoFile filein, unsigned int nBlockPos, CBlock* block);
extern bool getCBlocksTxByFilePos(CAutoFile filein, unsigned int nBlockPos, unsigned int txId, CTransaction& tx);
extern int dw_zip_block;

class CTransaction
{
...
    bool ReadFromDisk(CDiskTxPos pos, FILE** pfileRet=NULL)
    {
        CAutoFile filein = CAutoFile(OpenBlockFile(pos.nFile, 0, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION);
        if (!filein)
            return error("CTransaction::ReadFromDisk() : OpenBlockFile failed");

        if( dw_zip_block > 0 )
		{
			//if( fDebug ) printf("CTransaction::ReadFromDisk():: pos.nFile [%d], nBlockPos [%d], nTxPos [%d], pfileRet [%d] \n", pos.nFile, pos.nBlockPos, pos.nTxPos, pfileRet);
			getCBlocksTxByFilePos(filein, pos.nBlockPos, pos.nTxPos, *this);
		}else{
        // Read transaction
        if (fseek(filein, pos.nTxPos, SEEK_SET) != 0)
            return error("CTransaction::ReadFromDisk() : fseek failed");

        try {
            filein >> *this;
        }
        catch (std::exception &e) {
            return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__);
        }}

        // Return file pointer
        if (pfileRet)
        {
            if (fseek(filein, pos.nTxPos, SEEK_SET) != 0)
                return error("CTransaction::ReadFromDisk() : second fseek failed");
            *pfileRet = filein.release();
        }
        return true;
    }
...
}

class CBlock
{
...
    bool WriteToDisk(unsigned int& nFileRet, unsigned int& nBlockPosRet, bool bForceWrite = false)
    {
        // Open history file to append
        CAutoFile fileout = CAutoFile(AppendBlockFile(nFileRet), SER_DISK, CLIENT_VERSION);
        if (!fileout)
            return error("CBlock::WriteToDisk() : AppendBlockFile failed");

        // Write index header
        unsigned int nSize = fileout.GetSerializeSize(*this);

        int nSize2 = nSize;
		string sRzt;
        if( dw_zip_block > 0 )
        {
// compression blcok +++ 
			nSize = bitnet_pack_block(this, sRzt);  // nSize include 4 byte( block Real size )
// compression blcok +++ 
        }

        fileout << FLATDATA(pchMessageStart) << nSize;

        // Write block
        long fileOutPos = ftell(fileout);
        if (fileOutPos < 0)
            return error("CBlock::WriteToDisk() : ftell failed");
        nBlockPosRet = fileOutPos;

        if( dw_zip_block == 0 ){ fileout << *this; }
        else{
		    //if( fDebug ) printf("main.h Block.WriteToDisk:: nFileRet [%d], nBlockSize [%d], zipBlockSize [%d], nBlockPosRet = [%d] \n", nFileRet, nSize2, nSize, nBlockPosRet);
// compression blcok +++ 
		    if( nSize > 0 ){
				fileout.write(sRzt.c_str(), nSize);
			}
			sRzt.resize(0);
// compression blcok +++ 
        }

        // Flush stdio buffers and commit to disk before returning
        fflush(fileout);
        if( bForceWrite || (!IsInitialBlockDownload() || (nBestHeight+1) % 500 == 0) )
            FileCommit(fileout);

        return true;
    }

    bool ReadFromDisk(unsigned int nFile, unsigned int nBlockPos, bool fReadTransactions=true)
    {
        SetNull();
		unsigned int iPos = nBlockPos;
		if( dw_zip_block > 0 ){ iPos = 0; }

        // Open history file to read
        CAutoFile filein = CAutoFile(OpenBlockFile(nFile, iPos, "rb"), SER_DISK, CLIENT_VERSION);
        if (!filein)
            return error("CBlock::ReadFromDisk() : OpenBlockFile failed");
        if (!fReadTransactions)
            filein.nType |= SER_BLOCKHEADERONLY;

        // Read block
        try {
            if( dw_zip_block > 0 )
            {
				getCBlockByFilePos(filein, nBlockPos, this);
			}else{ filein >> *this; }
        }
        catch (std::exception &e) {
            return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__);
        }

        // Check the header
        if (fReadTransactions && IsProofOfWork() && !CheckProofOfWork(GetPoWHash(), nBits))
            return error("CBlock::ReadFromDisk() : errors in block header");

        return true;
    }

...
}

#include "lz4/lz4.h"
#include "lzma/LzmaLib.h"

int StreamToBuffer(CDataStream &ds, string& sRzt, int iSaveBufSize)
{
	int bsz = ds.size();
	int iRsz = bsz;
	if( iSaveBufSize > 0 ){ iRsz = iRsz + 4; }
	sRzt.resize(iRsz);
	char* ppp = (char*)sRzt.c_str();
	if( iSaveBufSize > 0 ){ ppp = ppp + 4; }
	ds.read(ppp, bsz);
	if( iSaveBufSize > 0 ){ *(unsigned int *)(ppp - 4) = bsz; }
	return iRsz;
}

int CBlockToBuffer(CBlock *pb, string& sRzt)
{
	CDataStream ssBlock(SER_DISK, CLIENT_VERSION);
	ssBlock << (*pb);
	int bsz = StreamToBuffer(ssBlock, sRzt, 0);
	return bsz;
}

int writeBufToFile(char* pBuf, int bufLen, string fName)
{
	int rzt = 0;
	std::ofstream oFs(fName.c_str(), std::ios::out | std::ofstream::binary);
	if( oFs.is_open() )
	{
		if( pBuf ) oFs.write(pBuf, bufLen);
		oFs.close(); 
		rzt++; 
	}
	return rzt;
}

int lz4_pack_buf(char* pBuf, int bufLen, string& sRzt)
{
	int worstCase = 0;
	int lenComp = 0;
    try{
		worstCase = LZ4_compressBound( bufLen );
		//std::vector<uint8_t> vchCompressed;   //vchCompressed.resize(worstCase);
		sRzt.resize(worstCase + 4);
		char* pp = (char *)sRzt.c_str();
		lenComp = LZ4_compress(pBuf, pp + 4, bufLen); 
		if( lenComp > 0 ){ *(unsigned int *)pp = bufLen;   lenComp = lenComp + 4; }
	}
    catch (std::exception &e) {
        printf("lz4_pack_buf err [%s]:: buf len %d, worstCase[%d], lenComp[%d] \n", e.what(), bufLen, worstCase, lenComp);
    }
	return lenComp;
}

int lz4_unpack_buf(const char* pZipBuf, unsigned int zipLen, string& sRzt)
{
	int rzt = 0;
	unsigned int realSz = *(unsigned int *)pZipBuf;
	if( fDebug )printf("lz4_unpack_buf:: zipLen [%d], realSz [%d],  \n", zipLen, realSz);
	sRzt.resize(realSz);
	char* pOutData = (char*)sRzt.c_str();
	
    // -- decompress
	rzt = LZ4_decompress_safe(pZipBuf + 4, pOutData, zipLen, realSz);
    if ( rzt != (int) realSz)
    {
            if( fDebug )printf("lz4_unpack_buf:: Could not decompress message data. [%d :: %d] \n", rzt, realSz);
            sRzt.resize(0);
    }
	return rzt;
}

int CBlockFromBuffer(CBlock* block, char* pBuf, int bufLen)
{
	CDataStream ssBlock(SER_DISK, CLIENT_VERSION);
	ssBlock.write(pBuf, bufLen);   int i = ssBlock.size();
	ssBlock >> (*block);
	return i;
}

int lz4_pack_block(CBlock* block, string& sRzt)
{
	int rzt = 0;
	string sbf;
	int bsz = CBlockToBuffer(block, sbf);
	if( bsz > 12 )
	{
		char* pBuf = (char*)sbf.c_str();
		rzt = lz4_pack_buf(pBuf, bsz, sRzt);
		//if( lzRzt > 0 ){ rzt = lzRzt; }  // + 4; }
	}
	sbf.resize(0);
	return rzt;
}

int lzma_depack_buf(unsigned char* pLzmaBuf, int bufLen, string& sRzt)
{
	int rzt = 0;
	unsigned int dstLen = *(unsigned int *)pLzmaBuf;
    sRzt.resize(dstLen);
	unsigned char* pOutBuf = (unsigned char*)sRzt.c_str();
    unsigned srcLen = bufLen - LZMA_PROPS_SIZE - 4;
	SRes res = LzmaUncompress(pOutBuf, &dstLen, &pLzmaBuf[LZMA_PROPS_SIZE + 4], &srcLen, &pLzmaBuf[4], LZMA_PROPS_SIZE);
	if( res == SZ_OK )//assert(res == SZ_OK);
	{
		//outBuf.resize(dstLen); // If uncompressed data can be smaller
		rzt = dstLen;
	}else sRzt.resize(0);
	if( fDebug ) printf("lzma_depack_buf:: res [%d], dstLen[%d],  rzt = [%d]\n", res, dstLen, rzt);
	return rzt;
}

int lzma_pack_buf(unsigned char* pBuf, int bufLen, string& sRzt, int iLevel, unsigned int iDictSize)  // (1 << 17) = 131072 = 128K
{
	int res = 0;
	int rzt = 0;
		unsigned propsSize = LZMA_PROPS_SIZE;
		unsigned destLen = bufLen + (bufLen / 3) + 128;
    try{
		sRzt.resize(propsSize + destLen + 4);
		unsigned char* pOutBuf = (unsigned char*)sRzt.c_str();

		res = LzmaCompress(&pOutBuf[LZMA_PROPS_SIZE + 4], &destLen, pBuf, bufLen, &pOutBuf[4], &propsSize,
                                      iLevel, iDictSize, -1, -1, -1, -1, -1);  // 1 << 14 = 16K, 1 << 16 = 64K
  
		//assert(propsSize == LZMA_PROPS_SIZE);
		//assert(res == SZ_OK);
		if( (res == SZ_OK) && (propsSize == LZMA_PROPS_SIZE) )  
		{
			//outBuf.resize(propsSize + destLen);
			*(unsigned int *)pOutBuf = bufLen; 
			rzt = propsSize + destLen + 4;
		}else sRzt.resize(0);
	
	}
    catch (std::exception &e) {
        printf("lzma_pack_buf err [%s]:: buf len %d, rzt[%d] \n", e.what(), bufLen, rzt);
    }
	if( fDebug ) printf("lzma_pack_buf:: res [%d], propsSize[%d], destLen[%d],  rzt = [%d]\n", res, propsSize, destLen, rzt);
	return rzt;
}

int lzma_pack_block(CBlock* block, string& sRzt, int iLevel, unsigned int iDictSize)  // (1 << 17) = 131072 = 128K
{
	int rzt = 0;
	string sbf;
	int bsz = CBlockToBuffer(block, sbf);
	if( bsz > 12 )
	{
		unsigned char* pBuf = (unsigned char*)sbf.c_str();
		rzt = lzma_pack_buf(pBuf, bsz, sRzt, iLevel, iDictSize);
		//if( lzRzt > 0 ){ rzt = lzRzt; }  // + 4; }
	}
	sbf.resize(0);
	return rzt;
}

int bitnet_pack_block(CBlock* block, string& sRzt)
{
	if( dw_zip_block == 1 )  return lzma_pack_block(block, sRzt, 9, uint_256KB);
	else if( dw_zip_block == 2 ) return lz4_pack_block(block, sRzt);
}

bool getCBlockByFilePos(CAutoFile filein, unsigned int nBlockPos, CBlock* block)
{
	bool rzt = false;
	int ips = nBlockPos - 4;  // get ziped block size;
	if (fseek(filein, ips, SEEK_SET) != 0)
		return error("getCBlockByFilePos:: fseek failed");
	filein >> ips;	// get ziped block size;
	if( fDebug )printf("getCBlockByFilePos:: ziped block size [%d] \n", ips);
	string s;   s.resize(ips);   char* pZipBuf = (char *)s.c_str();
	filein.read(pZipBuf, ips);
	string sUnpak;
	int iRealSz;
	if( dw_zip_block == 1 ) iRealSz = lzma_depack_buf((unsigned char*)pZipBuf, ips, sUnpak);
	else if( dw_zip_block == 2 ) iRealSz = lz4_unpack_buf(pZipBuf, ips - 4, sUnpak);
	if( fDebug )printf("getCBlockByFilePos:: ziped block size [%d], iRealSz [%d] \n", ips, iRealSz);
	if( iRealSz > 0 )
	{
		pZipBuf = (char *)sUnpak.c_str();
		rzt = CBlockFromBuffer(block, pZipBuf, iRealSz) > 12;
		/*if( fDebug ){
			if( block->vtx.size() < 10 )
			{
				printf("\n\n getCBlockByFilePos:: block info (%d): \n", rzt);
				block->print(); 
			}else printf("\n\n getCBlockByFilePos:: block vtx count (%d) is too large \n", block->vtx.size());
		}*/
	}
	s.resize(0);   sUnpak.resize(0);
	return rzt;
}

bool getCBlocksTxByFilePos(CAutoFile filein, unsigned int nBlockPos, unsigned int txId, CTransaction& tx)
{
	bool rzt = false;
	CBlock block;
	rzt = getCBlockByFilePos(filein, nBlockPos, &block);
	if( rzt )
	{
		if( block.vtx.size() > txId )
		{
			tx = block.vtx[txId];
			if( fDebug ){
			printf("\n\n getCBlocksTxByFilePos:: tx info: \n");
			tx.print(); }
		}else rzt = false;
	}
	return rzt;
}

FindBlockPos bool (&state CValidationState, &pos CDiskBlockPos, int nAddSize unsigned, int nHeight unsigned, nTime Uint64, fKnown bool = false)
{
...
        /* while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { */
        if( ((nHeight / 10000) > 0) && ((nHeight % 10000) == 0) ) {
            printf("nHeight = [%d], Leaving block file %i: %s\n", nHeight, nLastBlockFile, infoLastBlockFile.ToString().c_str());
            FlushBlockFile(true);
            nLastBlockFile++;
            infoLastBlockFile.SetNull();
            pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine
            fUpdatedLast = true;
        }
...
}