/* * @project: TERA * @version: Development (beta) * @license: MIT (not for evil) * @copyright: Yuriy Ivanov (Vtools) 2017-2019 [progr76@gmail.com] * Web: https://terafoundation.org * Twitter: https://twitter.com/terafoundation * Telegram: https://t.me/terafoundation */ "use strict"; import * as fs from 'fs' import CRest from './rest-loader' import { TYPE_TRANSACTION } from '../constant/account'; import { STreeBuffer } from './base'; //import * as crypto from 'crypto'; require('./block-loader-const'); const STAT_BLOCK_LOAD_PERIOD = global.CONSENSUS_PERIOD_TIME / 5; export default class CBlock extends CRest { MapMapLoaded BlockChain ChainID BlockID TaskNodeIndex LoadedChainList LastChainLoad StartLoadBlockTime LoadHistoryMode MapBlockBodyLoad BlockNumDB RelayMode LoadHistoryMessage LastLoadedBlockNum LoadHistoryContext LoadBlockStatNum BADHashCount ActualNodes constructor(SetKeyPair, RunIP, RunPort, UseRNDHeader, bVirtual) { super(SetKeyPair, RunIP, RunPort, UseRNDHeader, bVirtual) this.MapMapLoaded = {} this.BlockChain = {} this.ChainID = 0 this.BlockID = 0 this.TaskNodeIndex = 0 this.LoadedChainList = [] this.LastChainLoad = undefined this.StartLoadBlockTime = 0 this.LoadHistoryMode = false this.MapBlockBodyLoad = {} if (!global.ADDRLIST_MODE && !this.VirtualMode) { let Self = this; setTimeout(function() { Self.CheckStartedBlocks() setInterval(Self.LoopChainLoad.bind(Self), 100) setInterval(Self.LoopBlockLoad.bind(Self), 10) setInterval(Self.LoopBlockBodyLoad.bind(Self), 1 * 1000) }, 1000) } } StopNode() { global.glStopNode = true } GenesisBlockHeaderDB(Num) { if (Num < 0) return undefined; var Block: any = { BlockNum: Num, TreeHash: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], AddrHash: global.DEVELOP_PUB_KEY0, Hash: this.GetHashGenesis(Num), PowHash: this.GetHashGenesis(Num), PrevHash: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], SeqHash: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], SumHash: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], Comment1: "GENESIS", Comment2: "", TrCount: 0, TrDataPos: 0, TrDataLen: 0, }; Block.SeqHash = this.GetSeqHash(Block.BlockNum, Block.PrevHash, Block.TreeHash) Block.SumPow = 0 Block.bSave = true return Block; } CheckStartedBlocks() { this.FindStartBlockNum() if (this.UseTruncateBlockDB) this.TruncateBlockDB(this.UseTruncateBlockDB) var CurNum = global.GetCurrentBlockNumByTime(); if (CurNum <= this.BlockNumDB) { this.TruncateBlockDB(CurNum) } CurNum = this.GetMaxNumBlockDB() if (CurNum <= this.BlockNumDB) { this.TruncateBlockDB(CurNum) this.BlockNumDB = CurNum } if (this.BlockNumDB < global.BLOCK_PROCESSING_LENGTH2) this.CreateGenesisBlocks() if (fs.existsSync(global.GetCodePath("EXPERIMENTAL/_run.js"))) { require(global.GetCodePath("EXPERIMENTAL/_run.js")).Run() } this.LoadMemBlocksOnStart() } CreateGenesisBlocks() { global.ToLog("====CreateGenesisBlock====") var PrevArr = []; for (var n = 0; n < global.BLOCK_PROCESSING_LENGTH2; n++) { var Block = this.GenesisBlockHeaderDB(n); PrevArr[n] = Block this.WriteBlockDB(Block) } return PrevArr; } GetPrevHash(Block) { var startPrev = Block.BlockNum - global.BLOCK_PROCESSING_LENGTH2; var Sum = 0; var arr = []; for (var i = 0; i < global.BLOCK_PROCESSING_LENGTH; i++) { var PrevBlock = this.GetBlock(startPrev + i); if (PrevBlock && PrevBlock.bSave) { Sum = Sum + PrevBlock.SumPow arr.push(PrevBlock.Hash) } else { return undefined; } } var PrevHash = global.CalcHashFromArray(arr, true); return PrevHash; } GetPrevHashDB(Block) { var startPrev = Block.BlockNum - global.BLOCK_PROCESSING_LENGTH2; var arr = []; for (var i = 0; i < global.BLOCK_PROCESSING_LENGTH; i++) { var num = startPrev + i; var PrevBlock = this.ReadBlockHeaderDB(num); if (!PrevBlock || !PrevBlock.bSave) { global.ToError(" ERROR CALC BLOCK: " + Block.BlockNum + " - prev block not found: " + num) return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; } arr.push(PrevBlock.Hash) } var PrevHash = global.CalcHashFromArray(arr, true); return PrevHash; } StartSyncBlockchain(Node?, bSilent?, bCheckPoint?) { this.FREE_ALL_MEM_CHAINS() if (global.NO_HISTORY_MODE) { this.LoadHistoryMode = 0 return; } if (global.CREATE_ON_START && !global.LOCAL_RUN) return; if (!global.GrayConnect()) this.RelayMode = true else this.RelayMode = false if (!bSilent) this.RelayMode = true this.LoadHistoryMode = true this.LoadHistoryMessage = !bSilent this.LoadHistoryContext = { PrevBlockNum: - 1, Node: Node, BlockNum: this.BlockNumDB, MapSend: {}, Foward: 1, Pause: 0, DeltaBlockNum: 10, StartTimeHistory: Date.now(), MaxTimeOut: 30 * 1000 } if (!bSilent && !bCheckPoint && global.REST_START_COUNT) { this.CheckSyncRest() } if (!this.ActualNodes.size) { global.ToLog("There is no connections to other nodes") } else { if (this.LoadHistoryMessage) { global.ToLog("Start synchronization") } } } LoopSyncBlockchain() { if (!this.ActualNodes.size) return; var Context; if (this.LoadRestContext && this.LoadRestContext.Mode < 200) Context = this.LoadRestContext else Context = this.LoadHistoryContext if (Context.PrevBlockNum === Context.BlockNum) { var DeltaTime = Date.now() - Context.StartTimeHistory; if (DeltaTime > Context.MaxTimeOut) { global.ToLog("DETECT TIMEOUT LOAD") this.StartSyncBlockchain() return; } } else { Context.PrevBlockNum = Context.BlockNum Context.StartTimeHistory = Date.now() } if (Context.LoopSyncRest) { this.LoopSyncRest() return; } if (Context.Pause) { if (this.LoadedChainList.length) { return; } Context.Pause = 0 Context.BlockNum = this.BlockNumDB } var BlockDB = this.ReadBlockHeaderDB(Context.BlockNum); if (!BlockDB || this.BlockNumDB >= global.GetCurrentBlockNumByTime() - global.BLOCK_PROCESSING_LENGTH - 2) { this.LoadHistoryMode = false if (this.LoadHistoryMessage) global.ToLog("Finish synchronization") if (!BlockDB) return; } var Ret = this.GetNextNode(Context, Context.BlockNum, 1); if (Ret.Result) { var Node = Ret.Node; this.SendF(Node, { "Method": "GETBLOCKHEADER", "Context": Context, "Data": { Foward: 1, BlockNum: Context.BlockNum, Hash: BlockDB.SumHash } }) } } StartLoadBlockHeader(LoadHash, Num, StrInfo, bIsSum) { if (this.LoadHistoryMode) return; if (global.NO_HISTORY_MODE) return; this.StartLoadBlockTime = Date.now() if (Num > this.CurrentBlockNum + global.TIME_START_SAVE) { return; } bIsSum = bIsSum || false var Tree = this.GetHistoryTree("StartLoadBlockHeader"); if (Tree.find({ hash: LoadHash })) return false; Tree.insert({ hash: LoadHash }) var chain: any = { id: 0, Count: 16, BlockNum: Num, IsSum: bIsSum, Hash: LoadHash, time: undefined, FindBlockDB: false, LoadDB: false, LoadCountDB: 0, LoadSumDB: 0, LoadSum: 0, ParentChain: undefined, RootChain: undefined, BlockNumStart: Num, HashStart: LoadHash, IsSumStart: bIsSum, BlockHead: undefined, MapSend: {}, Comment2: "", StopSend: false, Info: "", Error: false, }; this.ChainBindMethods(chain) chain.AddInfo(StrInfo) this.SetChainNum(chain) var max = 3; while (max > 0) { max-- if (!this.SendChainNext(chain, false)) break; } return true; } SetChainNum(chain) { if (!chain.id) { this.ChainID++ chain.id = this.ChainID } this.LoadedChainList.push(chain) } LoopChainLoad() { if (global.glStopNode) return; if (this.UseTruncateBlockDB) this.TruncateBlockDB(this.UseTruncateBlockDB) if (this.LoadHistoryMode) { this.LoopSyncBlockchain() return; } if (this.BlockNumDB < this.CurrentBlockNum - global.BLOCK_PROCESSING_LENGTH2) { this.StartSyncBlockchain() return; } if (global.LOAD_TO_BEGIN && this.BlockNumDBMin) { this.SendLoadToBegin() } var CountStopSend = 0; var min_num = this.CurrentBlockNum - global.MAX_COUNT_CHAIN_LOAD; var min_num_load = this.CurrentBlockNum; for (var i = 0; i < this.LoadedChainList.length; i++) { var chain = this.LoadedChainList[i]; if (!chain) { this.LoadedChainList.splice(i, 1) continue; } var RootChain = chain.GetRootChain(); if (chain.RootChain) chain.RootChain = RootChain if (RootChain.BlockNum < min_num_load) min_num_load = RootChain.BlockNum if (!chain.StopSend) { if (chain.BlockHead) { if (chain.BlockNum < this.CurrentBlockNum - global.COUNT_HISTORY_BLOCKS_FOR_LOAD) { if (global.WATCHDOG_DEV) global.ToLog("Very long length of blocks to load history, stop chain with id=" + chain.id + " (" + chain.BlockNum + "-" + chain.BlockNumMax + ")") chain.StopSend = true chain.AddInfo("Stop load #1") this.FREE_ALL_MEM_CHAINS() return; } else if (chain.BlockNumMax < min_num) { if (global.WATCHDOG_DEV) global.ToLog("Timeout - stop load chain with id=" + chain.id + " (" + chain.BlockNum + "-" + chain.BlockNumMax + ")") chain.StopSend = true chain.AddInfo("Stop load #2") this.ClearChains(chain, 0) } } } if (chain && !chain.IsSum && !chain.StopSend) { var StrKey = "H:" + global.GetHexFromArr(chain.Hash); var Map = this.GetMapLoadedFromChain(chain); var WasBlock = Map[StrKey]; if (WasBlock && WasBlock.chain !== chain && global.CompareArr(WasBlock.Hash, chain.Hash) === 0 && !WasBlock.chain.Deleted) { if (global.WATCHDOG_DEV) global.ToLog("Was hash in chain " + WasBlock.chain.id + " - stop load chain with id=" + chain.id + " (" + chain.BlockNum + ")") chain.Comment2 = "was hash" chain.StopSend = true } } if (chain && chain.StopSend) { CountStopSend++ } chain = this.LoadedChainList[i] if (chain && !chain.GetFindDB() && !chain.StopSend) { this.SendChainNext(chain, true) } } global.ADD_TO_STAT("MAX:LOADEDCHAINLIST", this.LoadedChainList.length) this.FREE_MEM_CHAINS(min_num_load) this.LastLoadedBlockNum = 0 if (this.LoadedChainList.length > global.COUNT_HISTORY_BLOCKS_FOR_LOAD) { if (global.WATCHDOG_DEV) global.ToLog("LoadedChainList>COUNT_HISTORY_BLOCKS_FOR_LOAD -> FREE_ALL_MEM_CHAINS") this.FREE_ALL_MEM_CHAINS() } } GetNextNode(task, keyid?, checktime?, BlockNum?) { var CurTime = global.GetCurrentTime(0) - 0; if (checktime && task.time) { var Delta = CurTime - task.time; if (Delta < global.PACKET_ALIVE_PERIOD_NEXT_NODE) return { Result: false, timewait: true }; } task.time = undefined var StartI = 0; if (task.Node) StartI = - 1 var timewait = false; var arr = this.GetActualNodes(); arr.sort(function(a, b) { return b.BlockProcessCount - a.BlockProcessCount; }) if (arr.length > 40) arr.length = 40 for (var i = StartI; i < arr.length; i++) { var Node; if (i === - 1) { Node = task.Node task.Node = undefined } else { this.TaskNodeIndex++ Node = arr[this.TaskNodeIndex % arr.length] } if (Node.Active) { if (!Node.INFO || !Node.INFO.WasPing || Node.StopGetBlock || (Node.INFO.CheckPointHashDB && global.CHECK_POINT.BlockNum && global.CompareArr(global.CHECK_POINT.Hash, Node.INFO.CheckPointHashDB) !== 0)) { timewait = true continue; } if (BlockNum !== undefined && Node.INFO && BlockNum > Node.INFO.BlockNumDB) { timewait = true continue; } if (Node.TaskLastSend) { var Delta = CurTime - Node.TaskLastSend; if (Delta < global.PERIOD_GET_BLOCK) { timewait = true continue; } } var keysend = "" + Node.addrStr + ":" + keyid; if (task.MapSend[keysend]) continue; Node.TaskLastSend = CurTime task.time = CurTime return { Result: true, Node: Node, timewait: timewait }; } } if (!task.RestartGetNextNode) task.RestartGetNextNode = 0 if (!timewait && task.RestartGetNextNode < 3) { if (!task.LastTimeRestartGetNextNode) task.LastTimeRestartGetNextNode = 0 var Delta = Date.now() - task.LastTimeRestartGetNextNode; if (Delta > 3000) { task.RestartGetNextNode++ task.LastTimeRestartGetNextNode = Date.now() task.MapSend = {} return { Result: false, timewait: true }; } } return { Result: false, timewait: timewait }; } SendChainNext(chain, checktime) { var Ret = this.GetNextNode(chain, chain.BlockNum, checktime); if (Ret.Result) { if (!chain.Context) chain.Context = { Chain: chain } var Node = Ret.Node; this.SendF(Node, { "Method": "GETBLOCKHEADER", "Context": chain.Context, "Data": { Foward: 0, BlockNum: chain.BlockNum, Hash: chain.Hash, IsSum: chain.IsSum, Count: chain.Count } }) var DopStr = ""; if (chain.IsSum) DopStr = "SUM:" chain.AddInfo(chain.BlockNum + "/" + DopStr + this.GetStrFromHashShort(chain.Hash) + "->" + global.GetNodeStrPort(Node)) return true; } return false; } static GETBLOCKHEADER_F() { return "{\ Foward:byte,\ BlockNum:uint,\ Hash:hash,\ Count:uint,\ IsSum:byte\ }"; } static GETBLOCKHEADER100_F() { return "{\ BlockNum:uint,\ Hash:hash,\ Count:uint\ }"; } GetBlockArrFromBuffer_Load(BufRead, Info) { var BlockArr = global.GetBlockArrFromBuffer(BufRead, Info); if (BlockArr.length > 0 && BlockArr[0].BlockNum === global.BLOCK_PROCESSING_LENGTH2) BlockArr.unshift(this.ReadBlockHeaderDB(global.BLOCK_PROCESSING_LENGTH2 - 1)) return BlockArr; } RETBLOCKHEADER_FOWARD(Info, CurTime) { if (!Info.Context.Foward) return; var Context = this.LoadHistoryContext; Context.time = undefined var BufRead = global.BufLib.GetReadBuffer(Info.Data); var arr = this.GetBlockArrFromBuffer_Load(BufRead, Info); var arr2 = []; var bFindDB = 0; if (arr.length > 1) for (var i = 0; i < arr.length; i++) { var Block = arr[i]; if (!Block) return; if (Block.BlockNum === global.CHECK_POINT.BlockNum && !global.IsZeroArr(global.CHECK_POINT.Hash)) { if (global.CompareArr(global.CHECK_POINT.Hash, Block.Hash) !== 0) { break; } Context.FindCheckPoint = true } if (Block.BlockNum < this.BlockNumDB) { break; } if (!bFindDB) { var BlockDB = this.ReadBlockHeaderDB(Block.BlockNum); if (BlockDB && global.CompareArr(BlockDB.SumHash, Block.SumHash) === 0) { bFindDB = 1 arr2.push(Block) } else if (BlockDB && global.IsZeroArr(BlockDB.SumHash)) { bFindDB = 1 arr2.push(Block) } } else if (bFindDB) { arr2.push(Block) } } if (arr2.length > 1) { Context.WasLoadNum = 1 var chain = { id: 0, StopSend: 1, WriteToDBAfterLoad: 1 }; this.ChainBindMethods(chain) this.SetChainNum(chain) this.PrepareTransactionsForLoad(chain, arr2) Context.BlockNum = Block.BlockNum Context.Pause = 1 } else { if (!Context.WasLoadNum) { Context.BlockNum = Math.floor(Context.BlockNum - Context.DeltaBlockNum) Context.DeltaBlockNum = Context.DeltaBlockNum * 1.2 if (Context.BlockNum < global.BLOCK_PROCESSING_LENGTH2) Context.BlockNum = global.BLOCK_PROCESSING_LENGTH2 - 1 this.BlockNumDB = Context.BlockNum this.SetTruncateBlockDB(Context.BlockNum) } else { var keysend = "" + Info.Node.addrStr + ":" + Context.BlockNum; Context.MapSend[keysend] = 1 } } } RETBLOCKHEADER(Info, CurTime) { Info.Node.NextPing = global.MIN_PERIOD_PING if (Info.Context.Foward) return this.RETBLOCKHEADER_FOWARD(Info, CurTime); var chain = Info.Context.Chain; if (chain && !chain.StopSend && !chain.Deleted) { var BufRead = global.BufLib.GetReadBuffer(Info.Data); chain.time = undefined var arr = this.GetBlockArrFromBuffer_Load(BufRead, Info); if (arr.length <= 1) { var keysend = "" + Info.Node.addrStr + ":" + chain.BlockNum; chain.MapSend[keysend] = 1 chain.AddInfo("NO:" + global.GetNodeStrPort(Info.Node)) return; } chain.AddInfo("L=" + arr.length + " from:" + global.GetNodeStrPort(Info.Node)) var NextLoadBlock; var PrevBlock; for (var i = arr.length - 1; i >= 0; i--) { var Block = arr[i]; var StrKey = global.GetHexFromArr(Block.SumHash); var MapBlockLoaded = this.GetMapLoaded(Block.BlockNum); var BlockFind = MapBlockLoaded[StrKey]; if (BlockFind && BlockFind.chain !== chain && BlockFind.chain.Deleted) { delete MapBlockLoaded[StrKey] BlockFind = undefined } if (!chain.BlockHead) chain.BlockHead = Block if (!chain.BlockNumMax) chain.BlockNumMax = Block.BlockNum var PrevBlock0 = PrevBlock; if (BlockFind) { if (PrevBlock) { PrevBlock.BlockDown = BlockFind PrevBlock.Send = undefined } PrevBlock = BlockFind } else { if (PrevBlock) { PrevBlock.BlockDown = Block PrevBlock.Send = undefined } PrevBlock = Block } if (BlockFind && BlockFind.chain !== chain) { chain.ParentChain = BlockFind.chain chain.RootChain = BlockFind.chain.GetRootChain() if (chain.RootChain) chain.RootChain.BlockNumMax = chain.BlockHead.BlockNum chain.StopSend = true chain.AddInfo("StopSend - Find load Block") break; } else if (!BlockFind) { Block.chain = chain Block.Node = Info.Node var StrSumHash = global.GetHexFromArr(Block.SumHash); MapBlockLoaded[StrSumHash] = Block var StrHash = global.GetHexFromArr(Block.Hash); MapBlockLoaded["H:" + StrHash] = Block var StrTreeHash = global.GetHexFromArr(Block.TreeHash); MapBlockLoaded["TH:" + StrTreeHash] = Block var BlockDB = this.ReadBlockHeaderDB(Block.BlockNum); if (BlockDB) { Block.Power = global.GetPowPower(Block.PowHash) chain.LoadCountDB++ chain.LoadSumDB += BlockDB.Power chain.LoadSum += Block.Power if (global.CompareArr(BlockDB.SumHash, Block.SumHash) === 0) { Block.FindBlockDB = true Block.SumPow = BlockDB.SumPow chain.FindBlockDB = true chain.StopSend = true chain.AddInfo("BlockFind - Find Block in DB") NextLoadBlock = undefined break; } } NextLoadBlock = Block } } if (NextLoadBlock && !NextLoadBlock.chain.StopSend) { if (arr.length >= chain.Count) { chain.Count = chain.Count * 2 if (chain.Count > global.COUNT_BLOCKS_FOR_LOAD) chain.Count = global.COUNT_BLOCKS_FOR_LOAD } if (chain.LoadCountDB >= global.COUNT_BLOCKS_FOR_CHECK_POW) { if (chain.LoadSumDB - chain.LoadSum > global.MAX_DELTA_COUNT_SUM_FOR_LOAD) { var Str = "ERR LOADED SUM POW chains: SumDB > Sum loaded from: " + global.NodeInfo(Info.Node); chain.StopSend = true chain.AddInfo(Str) } } if (!chain.StopSend) this.BlockChainLoad(NextLoadBlock) } if (chain.GetFindDB()) this.CheckToStartLoadBlockData(chain) } } BlockChainLoad(Block) { var chain = Block.chain; Block.Send = undefined chain.BlockNum = Block.BlockNum chain.Hash = Block.SumHash chain.IsSum = true chain.StopSend = false chain.FindBlockDB = false chain.RootChain = undefined chain.ParentChain = undefined chain.AddInfo("SetChainSend:" + chain.BlockNum) } CheckToStartLoadBlockData(chain) { if (chain.Deleted) return; var arr = this.GetArrFromChain(chain); if (arr.length < 2) return; var BlockMax = arr[arr.length - 1]; var BlockMin = arr[0]; var PrevBlock = BlockMin; for (var i = 1; i < arr.length; i++) { var Block = arr[i]; Block.Power = global.GetPowPower(Block.PowHash) Block.SumPow = PrevBlock.SumPow + Block.Power PrevBlock = Block } var BlockNow = this.ReadBlockHeaderDB(BlockMax.BlockNum); if (BlockNow && (BlockMax.SumPow < BlockNow.SumPow || BlockMax.SumPow === BlockNow.SumPow && global.CompareArr(BlockMax.PowHash, BlockNow.PowHash) < 0)) { var Str = "Low SumPow"; chain.AddInfo(Str) return; } var Str = "Start Load blocks: " + (BlockMin.BlockNum + 1) + " - " + BlockMax.BlockNum; chain.AddInfo(Str) this.PrepareTransactionsForLoad(chain, arr) } GetArrFromChain(chain) { var arr = []; var Block = chain.BlockHead; while (Block) { arr.unshift(Block) if (Block.AddToLoad || Block.FindBlockDB || Block.LoadDBFinaly) { break; } Block = Block.BlockDown } return arr; } PrepareTransactionsForLoad(chain, arr, bNoSlice?) { if (!bNoSlice) arr = arr.slice(1) chain.arr = arr if (arr.length > 0) { for (var i = 0; i < arr.length; i++) arr[i].AddToLoad = 1 chain.CurNumArrLoad = 0 } } LoopBlockLoad() { if (global.glStopNode) return; var CountSend = 0; for (var num = 0; num < this.LoadedChainList.length; num++) { var chain = this.LoadedChainList[num]; if (chain && chain.arr && chain.arr.length && chain.StopSend) { var Count = 0; for (var i = chain.CurNumArrLoad; i < chain.arr.length; i++) { Count++ var Block = chain.arr[i]; if (!global.IsZeroArr(Block.TreeHash) && !Block.TreeEq && !Block.LoadDBFinaly) { if (!Block.MapSend) { if (!Block.BodyLoad) { var BlockDB = this.ReadBlockHeaderDB(Block.BlockNum); if (BlockDB) { if (global.CompareArr(BlockDB.TreeHash, Block.TreeHash) == 0) { Block.TreeEq = true Block.Reserv500 = BlockDB.Reserv500 Block.TrDataPos = BlockDB.TrDataPos Block.TrDataLen = BlockDB.TrDataLen continue; } } } Block.MapSend = {} } if (this.SendBlockNext(Block)) { CountSend++ if (CountSend >= global.MAX_BLOCK_SEND) return; } } else { if (i === chain.CurNumArrLoad) { chain.CurNumArrLoad++ Block.LoadDBFinaly = true Count = 0 } } } this.CheckAndWriteLoadedChain(chain) } } } CheckAndWriteLoadedChain(chain) { if (chain.CurNumArrLoad >= chain.arr.length) { var Block = chain.arr[chain.arr.length - 1]; if (chain.WriteToDBAfterLoad || Block.BlockNum >= this.CurrentBlockNum + global.TIME_START_SAVE - 2) { var bAllLoaded = true; if (!chain.WriteToDBAfterLoad) { var cur_parent = chain.ParentChain; while (cur_parent) { if (cur_parent.arr && cur_parent.CurNumArrLoad < cur_parent.arr.length) { bAllLoaded = false break; } cur_parent = cur_parent.ParentChain } } if (bAllLoaded) { var arr = []; var cur_chain = chain; while (cur_chain) { if (cur_chain.arr) for (var i = cur_chain.arr.length - 1; i >= 0; i--) { var Block = cur_chain.arr[i]; arr.unshift(Block) } cur_chain = cur_chain.ParentChain } this.WriteLoadedBlockArr(arr) } } } } WriteLoadedBlockArr(arr) { if (!arr.length) return; var startTime = process.hrtime(); if (this.LoadHistoryMessage) global.ToLog("WRITE DATA Count:" + arr.length + " " + arr[0].BlockNum + "-" + arr[arr.length - 1].BlockNum, 2) var CurrentBlockNum = global.GetCurrentBlockNumByTime(); var Block, FirstBlock; for (var i = 0; i < arr.length; i++) { Block = arr[i] if (Block.BlockNum > this.BlockNumDB + 1) break; if (!FirstBlock) FirstBlock = Block Block.BlockDown = undefined if (Block.BlockNum > this.BlockNumDBMin + global.BLOCK_PROCESSING_LENGTH2) { var PrevHash = this.GetPrevHashDB(Block); if (global.CompareArr(PrevHash, Block.PrevHash) !== 0) { if (global.WATCHDOG_DEV) global.ToError("******** ERROR LOADED DATA Count:" + arr.length + " AT BLOCK NUM:" + Block.BlockNum + " (" + arr[0].BlockNum + "-" + arr[arr.length - 1].BlockNum + ")") this.FREE_ALL_MEM_CHAINS() return; } } var Res: any = 0; if (Block.TreeEq) { this.ReadBlockBodyDB(Block) Res = this.WriteBlockDBFinaly(Block) } else { if (global.IsZeroArr(Block.TreeHash)) { Res = this.WriteBlockDB(Block) } else { global.ToLogTrace("global.IsZeroArr(Block.TreeHash)") throw "global.IsZeroArr(Block.TreeHash)"; } } if (!Res) { global.ToLog("ERROR WRITE DB, NUM=" + Block.BlockNum) this.FREE_ALL_MEM_CHAINS() return; } Block.LoadDB = true if (Block.BlockNum >= CurrentBlockNum - global.BLOCK_COUNT_IN_MEMORY) { this.CopyBlockToMem(Block) } else { if (Block.arrContent) Block.arrContent.length = 0 Block.arrContent = undefined } var BlockMem = this.BlockChain[Block.BlockNum]; if (BlockMem) { global.AddInfoBlock(BlockMem, "LOAD:" + global.GetPowPower(Block.PowHash) + " TH:" + this.GetStrFromHashShort(Block.TreeHash).substr(0, 4)) } } if (Block && FirstBlock) { var CurNumStart = Math.max(FirstBlock.BlockNum + 8, Block.BlockNum + 1); this.SetNoPOW(CurNumStart, 1, FirstBlock.BlockNum) } this.FREE_ALL_MEM_CHAINS() global.ADD_TO_STAT_TIME("WRITECHAIN_TO_DB_TIME", startTime) } SetNoPOW(BlockNumFrom, bReload, RefBlockNum) { //defiend in block-exchange.js } CopyBlock(Block, BlockDst) { BlockDst.BlockNum = Block.BlockNum BlockDst.TreeHash = Block.TreeHash BlockDst.AddrHash = Block.AddrHash BlockDst.PrevHash = Block.PrevHash BlockDst.SumHash = Block.SumHash BlockDst.SumPow = Block.SumPow BlockDst.TrDataPos = Block.TrDataPos BlockDst.TrDataLen = Block.TrDataLen BlockDst.SeqHash = Block.SeqHash BlockDst.Hash = Block.Hash BlockDst.PowHash = Block.PowHash BlockDst.TrCount = Block.TrCount BlockDst.arrContent = Block.arrContent BlockDst.bSave = Block.bSave } CopyBlockToMem(Block) { var BlockMem = this.BlockChain[Block.BlockNum]; if (BlockMem) { this.CopyBlock(Block, BlockMem) BlockMem.Prepared = true BlockMem.MinTrPow = undefined this.RecreateMaxPOW(BlockMem) } this.AddToStatBlockConfirmation(Block) } RecreateMaxPOW(Block) { //defiend in block-exchange.js } ClearMaxInBlock(Block) { Block.MaxPOW = {} var POW = Block.MaxPOW; POW.SeqHash = Block.SeqHash POW.AddrHash = Block.AddrHash POW.PrevHash = Block.PrevHash POW.TreeHash = Block.TreeHash POW.Hash = Block.Hash POW.PowHash = Block.PowHash POW.SumPow = Block.SumPow Block.MaxSum = {} POW = Block.MaxSum POW.SeqHash = Block.SeqHash POW.AddrHash = Block.AddrHash POW.PrevHash = Block.PrevHash POW.TreeHash = Block.TreeHash POW.Hash = Block.Hash POW.PowHash = Block.PowHash POW.SumHash = Block.SumHash POW.SumPow = Block.SumPow } AddToStatBlockConfirmation(Block) { if (Block.BlockNum > global.START_BLOCK_RUN + global.BLOCK_PROCESSING_LENGTH2) { var TimeDelta = this.CurrentBlockNum - Block.BlockNum; global.ADD_TO_STAT("MAX:BlockConfirmation", TimeDelta) } else { global.ADD_TO_STAT("MAX:BlockConfirmation", global.BLOCK_PROCESSING_LENGTH) } } SendBlockNext(Block) { var SendResult = 0; var Key = global.GetHexFromArr(Block.TreeHash); while (true) { var Ret = this.GetNextNode(Block, Key, true, Block.BlockNum); if (Ret.Result) { var Node = Ret.Node; if (!Block.Context) Block.Context = { Block: Block } this.SendF(Node, { "Method": "GETBLOCK", "Context": Block.Context, "Data": { BlockNum: Block.BlockNum, TreeHash: Block.TreeHash } }) Node.SendBlockCount++ SendResult = 1 global.AddInfoBlock(Block, "SendNext") if (Block.chain) Block.chain.AddInfo("QUERY BL:" + Block.BlockNum + "/" + this.GetStrFromHashShort(Block.TreeHash) + " TO:" + global.GetNodeStrPort(Node)) } else { if (!Ret.timewait) { this.ClearChains(Block.chain, true) } } break; } return SendResult; } ClearChains(DeleteChain, bShow?) { if (!DeleteChain) { this.FREE_ALL_MEM_CHAINS() return this.LoadedChainList.length; } var allsum = this.LoadedChainList.length; var Sum = 0; for (var i = 0; i < this.LoadedChainList.length; i++) { var chain = this.LoadedChainList[i]; if (chain) { if (chain === DeleteChain) { chain.Deleted = true this.LoadedChainList[i] = undefined Sum++ } if (chain.ParentChain === DeleteChain) { Sum += this.ClearChains(chain) } } } if (bShow) { global.ToLog("===========ClearChains================= " + DeleteChain.id + " count=" + Sum + "/" + allsum) } return Sum; } RecalcLoadBlockStatictic() { return; var TimeNum = Math.floor(Date.now() / STAT_BLOCK_LOAD_PERIOD); if (this.LoadBlockStatNum === TimeNum) return; this.LoadBlockStatNum = TimeNum const PeriodSec = 5; const Period = global.CONSENSUS_PERIOD_TIME / STAT_BLOCK_LOAD_PERIOD; const PeriodCount = PeriodSec * Period; var FreeGet = 64; var it = this.ActualNodes.iterator(), Node; while ((Node = it.next()) !== null) { var arr = Node.SendBlockArr; arr.push(Node.SendBlockCount) if (arr.length > PeriodCount) { arr.shift() } arr = Node.LoadBlockArr arr.push(Node.LoadBlockCount) if (arr.length > PeriodCount) { arr.shift() } var SendPackets = 0; var LoadPackets = 0; for (var i = 0; i < Node.SendBlockArr.length; i++) SendPackets += Node.SendBlockArr[i] for (var i = 0; i < Node.LoadBlockArr.length; i++) LoadPackets += Node.LoadBlockArr[i] Node.SendBlockCountAll = SendPackets Node.LoadBlockCountAll = LoadPackets var Nuts = Math.floor(LoadPackets / PeriodSec); var RestPackets = SendPackets - LoadPackets; var CountGet = 1 + Math.floor(Math.max(0, (Nuts - RestPackets / Period))); Node.CanGetBlocks = Math.min(FreeGet, CountGet) FreeGet -= Node.CanGetBlocks Node.SendBlockCount = 0 Node.LoadBlockCount = 0 global.ADD_TO_STAT("NODE_CAN_GET:" + global.NodeName(Node), Node.CanGetBlocks, 1) } } static GETBLOCK_F() { return "{\ BlockNum:uint,\ TreeHash:hash,\ }"; } RETGETBLOCK(Info, CurTime) { Info.Node.NextPing = global.MIN_PERIOD_PING var Block = Info.Context.Block; if (Block && !Block.TreeEq) { var Data = global.BufLib.GetObjectFromBuffer(Info.Data, global.FORMAT_BLOCK_TRANSFER, global.WRK_BLOCK_TRANSFER); Info.Data = undefined if (Data.BlockNum !== Block.BlockNum || global.CompareArr(Data.TreeHash, Block.TreeHash) !== 0) { this.SetBlockNOSendToNode(Block, Info.Node, "NO") return; } if (Block.chain) { Block.chain.AddInfo("Load TR:" + Data.BlockNum + "/" + this.GetStrFromHashShort(Data.TreeHash) + " from:" + global.GetNodeStrPort(Info.Node)) global.AddInfoBlock(Block, "LOAD TR OK") } var arrContent = Data.arrContent; var TreeHash = global.CalcTreeHashFromArrBody(Block.BlockNum, arrContent); if (global.CompareArr(Block.TreeHash, TreeHash) !== 0) { global.ToLog("2. BAD CMP TreeHash block=" + Block.BlockNum + " from:" + global.NodeName(Info.Node) + " TreeHash=" + global.GetHexFromArr(TreeHash) + " BlockTreeHash=" + global.GetHexFromArr(Block.TreeHash)) this.SetBlockNOSendToNode(Block, Info.Node, "BAD CMP TreeHash") return; } if (arrContent.length > 0 && Data.BlockNum % global.PERIOD_ACCOUNT_HASH === 0) { var TR = arrContent[0]; if (TR[0] === TYPE_TRANSACTION.TYPE_TRANSACTION_ACC_HASH) { if (!global.DApps.Accounts.TRCheckAccountHash(TR, Data.BlockNum)) { global.ToLog(`global.DApps.Accounts.TRCheckAccountHash(TR, Data.BlockNum) ${JSON.stringify(TR)}, ${Data.BlockNum}`) if (!this.BADHashCount) this.BADHashCount = 0 this.BADHashCount++ global.ToLog("**** BAD ACCOUNT Hash in block=" + Block.BlockNum + " from:" + global.NodeName(Info.Node) + " ****") global.ToLog("May be need to Rewrite transactions from: " + (Block.BlockNum - 2 * global.DELTA_BLOCK_ACCOUNT_HASH)) this.SetBlockNOSendToNode(Block, Info.Node, "BAD CMP ACC HASH") if (global.WATCHDOG_BADACCOUNT && this.BADHashCount > 60) { global.ToLog("Run WATCHDOG!") this.BADHashCount = 0 this.FREE_ALL_MEM_CHAINS() this.SetTruncateBlockDB(Block.BlockNum - 5 * global.DELTA_BLOCK_ACCOUNT_HASH) } else { } return; } } } Block.arrContent = arrContent var Ret = this.WriteBodyDB(Block); Block.TrCount = 0 Block.arrContent.length = 0 Block.arrContent = undefined if (!Ret) { this.SetBlockNOSendToNode(Block, Info.Node, "Error write") return; } Block.TreeEq = true Block.Send = undefined global.ADD_TO_STAT("BLOCK_LOADED", 1) Info.Node.LoadBlockCount++ if (global.GrayConnect()) Info.Node.BlockProcessCount++ if (this.LoadHistoryMode) { var Context = this.LoadHistoryContext; Context.PrevBlockNum = Context.BlockNum Context.StartTimeHistory = Date.now() } } } SendCanBlock(Node, Block) { Node.SendBlockCount++ if (!Node.INFO.BlockNumDB) return; if (Node.INFO.BlockNumDB >= Block.BlockNum) { this.SendF(Node, { "Method": "CANBLOCK", "Data": { BlockNum: Block.BlockNum } }) } } static CANBLOCK_F() { return "{BlockNum:uint}"; } CANBLOCK(Info, CurTime) { var Data = this.DataFromF(Info); this.SendF(Info.Node, { "Method": "RETCANBLOCK", "Data": { Result: 1 } }) } static RETCANBLOCK_F() { return "{Result:byte}"; } RETCANBLOCK(Info, CurTime) { var Data = this.DataFromF(Info); if (Data.Result === 1) { Info.Node.LoadBlockCount++ } } DataFromF(Info, bSendFormat?): any { //defiend in server.js } SetBlockNOSendToNode(Block, Node, Str) { var Str = global.GetHexFromArr(Block.TreeHash); var Str2 = this.GetStrFromHashShort(Block.TreeHash); var keysend = "" + Node.addrStr + ":" + Str; Block.MapSend[keysend] = 1 if (Block.chain) Block.chain.AddInfo("" + Block.BlockNum + " " + Str2 + "<-" + global.GetNodeStrPort(Node)) } FindBlockInLoadedChain(BlockNum, TreeHash) { var StrTreeHash = global.GetHexFromArr(TreeHash); var MapBlockLoaded = this.GetMapLoaded(BlockNum); var BlockFind = MapBlockLoaded["TH:" + StrTreeHash]; if (BlockFind && BlockFind.TreeEq) return BlockFind; else return undefined; } CheckSeqHashDB(Block, StrError) { if (Block.BlockNum < global.BLOCK_PROCESSING_LENGTH2) return true; var TreeHashTest = global.CalcTreeHashFromArrBody(Block.BlockNum, Block.arrContent); if (global.CompareArr(TreeHashTest, Block.TreeHash) !== 0) { var StrHex = global.GetHexFromArr(TreeHashTest); var StrHex0 = global.GetHexFromArr(Block.TreeHash); var Str = StrError + " #3 ERROR TREEHASH: " + Block.BlockNum + " Hex:" + StrHex0.substr(0, 12) + " != " + StrHex.substr(0, 12); if (global.WATCHDOG_DEV) global.ToErrorTrace(Str) else global.ToError(Str) return false; } var PrevHash = this.GetPrevHashDB(Block); var testSeqHash = this.GetSeqHash(Block.BlockNum, PrevHash, Block.TreeHash); var TestValue = global.GetHashFromSeqAddr(testSeqHash, Block.AddrHash, Block.BlockNum, PrevHash); if (global.CompareArr(TestValue.Hash, Block.Hash) !== 0) { var Str = StrError + " #2 ERROR HASH - block num: " + Block.BlockNum; if (global.WATCHDOG_DEV) global.ToErrorTrace(Str) else global.ToError(Str) return false; } return true; } ToLogBlock(Block, StrInfo, arr) { global.ToLog("-------------" + StrInfo) global.ToLog("BlockNum=" + (Block.BlockNum)) global.ToLog("Hash=" + global.GetHexFromArr(Block.Hash)) global.ToLog("SeqHash=" + global.GetHexFromArr(Block.SeqHash)) global.ToLog("PrevHash=" + global.GetHexFromArr(Block.PrevHash)) global.ToLog("TreeHash=" + global.GetHexFromArr(Block.TreeHash)) global.ToLog("AddrHash=" + global.GetHexFromArr(Block.AddrHash)) global.ToLog("SumHash=" + global.GetHexFromArr(Block.SumHash)) global.ToLog("SumPow=" + Block.SumPow) for (var i = 0; i < arr.length; i++) { global.ToLog("arr[" + i + "]=" + global.GetHexFromArr(arr[i])) } } GetBlock(num, bToMem?, bReadBody?) { if (bToMem === undefined) bToMem = true if (num < this.CurrentBlockNum - global.BLOCK_COUNT_IN_MEMORY) bToMem = false var Block = this.BlockChain[num]; if (!Block) { if (bReadBody) Block = this.ReadBlockDB(num) else Block = this.ReadBlockHeaderDB(num) if (bToMem) { this.BlockChain[num] = Block } } return Block; } GetMapLoaded(num) { if (num < 0) num = 0 var index = Math.floor(num / global.BLOCK_COUNT_IN_MEMORY); var map = this.MapMapLoaded[index]; if (!map) { map = {} this.MapMapLoaded[index] = map } return map; } GetMapLoadedFromChain(chain) { return this.GetMapLoaded(chain.BlockNumStart); } FREE_MEM_BLOCKS(NumMax) { for (var key in this.BlockChain) { var Block = this.BlockChain[key]; if (!Block || Block.BlockNum < NumMax) { delete this.BlockChain[key] } } } FREE_MEM_CHAINS(NumMax) { this.FREE_MEM_BLOCKS(NumMax - global.BLOCK_COUNT_IN_MEMORY) var maxArrMap = Math.floor(NumMax / global.BLOCK_COUNT_IN_MEMORY) - 1; if (maxArrMap >= 0) { var nWasCount = 0; for (var key in this.MapMapLoaded) if (key as any < maxArrMap) { nWasCount++ delete this.MapMapLoaded[key] } } } FREE_ALL_MEM_CHAINS() { this.FREE_MEM_BLOCKS(this.BlockNumDB - global.BLOCK_COUNT_IN_MEMORY) for (var i = 0; i < this.LoadedChainList.length; i++) { var chain = this.LoadedChainList[i]; if (chain) { chain.Deleted = true chain.ChainMax = undefined } } if (!this.LoadHistoryMode) { this.AddValueToHistory("LoadedChainList", this.LoadedChainList) this.AddValueToHistory("MapMapLoaded", this.MapMapLoaded) } this.LoadedChainList = [] this.MapMapLoaded = {} //@ts-ignore if (typeof gc === "function") //@ts-ignore gc() } AddValueToHistory(typedata, val) { var Arr = global.HistoryBlockBuf.LoadValue(typedata, 1); if (!Arr) { Arr = [] global.HistoryBlockBuf.SaveValue(typedata, Arr) } Arr.push(val) } GetHistoryTree(typedata) { var Tree = global.HistoryBlockBuf.LoadValue(typedata, 1); if (!Tree) { Tree = new global.RBTree(global.CompareItemHash) global.HistoryBlockBuf.SaveValue(typedata, Tree) } return Tree; } ChainBindMethods(chain) { function GetRootChain() { var Count = 0; var root_chain = this; while (root_chain.RootChain) { Count++ root_chain = root_chain.RootChain if (Count > global.MAX_COUNT_CHAIN_LOAD) { global.TO_ERROR_LOG("BLOCK", 10, "Error COUNT GetRootChain") global.SERVER.FREE_ALL_MEM_CHAINS() return undefined; } } return root_chain; }; function GetFindDB() { var Root = this.GetRootChain(); if (Root) return Root.FindBlockDB; else return false; }; chain.GetRootChain = GetRootChain.bind(chain) chain.GetFindDB = GetFindDB.bind(chain) chain.AddInfo = global.AddInfoChain.bind(chain) } GetMemoryStamp(Str) { return Str + ":##:" + Math.floor(this.CurrentBlockNum / global.BLOCK_COUNT_IN_MEMORY); } GetStrFromHashShort(Hash) { var Str = global.GetHexFromArr(Hash); if (typeof Str === "string") return Str.substr(0, 6); else return ""; } ToLogTime(startTime, Str) { const Time = process.hrtime(startTime); var deltaTime = (Time[0] * 1000 + Time[1] / 1e6); global.ToLog(Str + " : " + deltaTime + "ms") } AddBlockToLoadBody(Block) { if (!this.MapBlockBodyLoad[Block.BlockNum]) { this.MapBlockBodyLoad[Block.BlockNum] = Block } } LoopBlockBodyLoad() { var arr = []; for (var key in this.MapBlockBodyLoad) { var Block = this.MapBlockBodyLoad[key]; if (!Block.BodyLoad) { Block.BodyLoad = 1 arr.push(Block) } } this.MapBlockBodyLoad = {} if (!arr.length) return; var chain = { StopSend: 1, WriteToDBAfterLoad: 1, BodyLoad: 1 }; this.ChainBindMethods(chain) this.SetChainNum(chain) this.PrepareTransactionsForLoad(chain, arr, 1) } }; global.LoadBlockFromNetwork = function(Params, F) { var BlockNum = Params.BlockNum; if (BlockNum >= global.SERVER.BlockNumDBMin) { global.ToLog("Cant LoadBlockFromNetwork:" + BlockNum, 2); F(1); return; } global.ToLog("Start DownloadBlockFromNetwork:" + BlockNum, 2); var TaskLoadBlockFromNetwork = { MapSend: {} }; var Ret = global.SERVER.GetNextNode(TaskLoadBlockFromNetwork, BlockNum, 1); if (Ret.Result) { let Node = Ret.Node; global.SERVER.SendF(Node, { "Method": "GETBLOCK", "Data": { BlockNum: BlockNum, TreeHash: [] }, "Context": { F: function(Info) { var Block = global.BufLib.GetObjectFromBuffer(Info.Data, global.FORMAT_BLOCK_TRANSFER, global.WRK_BLOCK_TRANSFER); Info.Data = undefined; if (!Block.BlockNum || Block.BlockNum !== Params.BlockNum) { global.ToLog("Error get BlockNum:" + Params.BlockNum + " from " + global.NodeName(Info.Node), 2); F(1); return; } global.ToLog("Got BlockFromNetwork:" + Params.BlockNum + " from " + global.NodeName(Info.Node), 2); var ResError; if (!Block.arrContent || Block.arrContent.length === 0) { ResError = 1; } else { ResError = 0; global.SERVER.WriteBlockDB(Block); } F(ResError, Block); } }, }); } else { global.ToLog("Not find node for download block", 2); F(1); } }; global.HistoryBlockBuf = new STreeBuffer(global.HISTORY_BLOCK_COUNT * 1000, global.CompareItemHashSimple, "string");