Utilitaire pour indexer des informations de la blockchain et dessiner des Plots. Focalisé sur la toile de confiance. http://datajune.coinduf.eu/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

234 lines
8.7 KiB

module BlockchainBrowser
export sync, browse
using ..Network, ..Config
using OrderedCollections: delete!
using JSON, Logging, ProgressMeter, OrderedCollections, Serialization
const CHUNK_FORMAT = "duniter" # duniter or dex
"""
sync()
download missing chunks as dex json archive
"""
function sync()
@info "checking current block number"
remote_current_block = get_current_block()
@info "remote block found: $remote_current_block"
remote_chunk_number = block_to_chunk(remote_current_block)-1
local_chunk_number = local_chunk()
@info "$local_chunk_number/$remote_chunk_number chunks available locally"
@showprogress "Downloading chunks... " for i in local_chunk_number:remote_chunk_number
try download_chunk(CHUNK_PATH, i)
catch; println(); @warn "can not get chunk number $i and higher"; break; end
end
end
"""
sync_by_block()
download missing chunks block by block to be up to date (if export is not up to date)
⚠️ this makes a lot of requests (one by block) and can be rejected by the server
"""
function sync_by_block()
@info "checking current block number"
remote_current_block = get_current_block()
@info "remote block found: $remote_current_block"
remote_chunk_number = block_to_chunk(remote_current_block)-1
local_chunk_number = local_chunk()
@info "$local_chunk_number/$remote_chunk_number chunks available locally"
if local_chunk_number < remote_chunk_number
@info "syncing block by block at $BMA_URL"
end
for i in local_chunk_number+1:remote_chunk_number
@info "downloading chunk $i"
chunk = Dict("blocks" => [])
@showprogress "Downloading blocks " for j in 1:CHUNK_SIZE
sleep(0.5) # sleep a bit to stay below quotas
block = get_block(i*CHUNK_SIZE+j-1)
push!(chunk["blocks"], block) # adds block to list
end
# print chunk to file when complete
chunkfile = joinpath(CHUNK_PATH, "chunk_$(i)-250.json")
open(chunkfile, "w") do io
JSON.print(io, chunk, 1)
end
end
end
"retains some info about the sampled blocks"
struct PartialBlock
version::Int # protocol version
number::Int # number of the block
powMin::Int # minimal pow difficulty
time::Int # unix time
medianTime::Int # median time computed on 24 last block
membersCount::Int # number of members
pseudoCount::Int # number of pseudo from beginning (not in blockchain)
monetaryMass::Int # monetary mass
issuersCount::Int # number of issuers in window
issuersFrame::Int # size of window
end
"struct to store info of blockchain"
mutable struct Storage
firstblock_timestamp::Int # timestamp of block zero
lastblock::Int # number of last block used to fill storage (for caching)
lastblock_timestamp::Int # timestamp of last block (for caching)
pseudos::Vector{String} # pseudo at position id
pubkeys::OrderedDict{String, Int} # pubkey -> id
cert_in::Vector{Tuple{Int, Tuple{Int, Int}}} # entering certs: (mediantime, (from, to))
cert_out::Vector{Tuple{Int, Tuple{Int, Int}}} # expiring certs: (mediantime, (from, to))
cert_exp::Dict{Tuple{Int, Int}, Vector{Int}} # an edge to all cert expiry dates: (from, to) -> [mediantime...]
joiners::Vector{Tuple{Int, Int}} # joiners: (blocknumber, id) (reasons: enough certs, expired membership renewal)
leavers::Vector{Tuple{Int, Int}} # leavers: (blocknumber, id) (reasons: not enough cert, revocation, membership expiry)
sampled_blocks::Vector{PartialBlock} # list of some blocks params sampled at SAMPLING_PERIOD
end
"default constructor"
Storage() = Storage(0,-1,0,[],OrderedDict(),[],[],Dict(),[],[],[])
"load last version of storage or return default"
function load_storage()
if isfile(STORAGE_CACHE_FILE)
@debug "found storage cache"
return deserialize(STORAGE_CACHE_FILE)
else
@debug "storage cache not found"
return Storage()
end
end
"show override"
function Base.show(io::IO, s::Storage)
print(io, "$(typeof(s)) with $(length(s.pseudos)) identities")
end
"get joiners in block"
function get_joiners(storage::Storage, block)
n = block["number"]
for join in block["joiners"]
parts = split(join, ":")
pubkey = parts[1]
pseudo = parts[5]
if pubkey in keys(storage.pubkeys)
id = storage.pubkeys[pubkey]
else
push!(storage.pseudos, pseudo)
id = length(storage.pseudos)
storage.pubkeys[pubkey] = id
end
push!(storage.joiners, (n, id))
end
end
"get leavers in block"
function get_leavers(storage::Storage, block)
n = block["number"]
# block["leavers"] expired membership (not necessarily exculded)
# block["revoked"] (will appear in exculded)
# excluded because of distance rule or missing certifications or revocation
for leav in block["excluded"]
push!(storage.leavers, (n, storage.pubkeys[leav]))
end
end
"new certifications in block"
function get_new_certs(storage, block)
t = block["medianTime"]
expiration = t + CERTVALIDITY
if !isempty(block["certifications"])
for cert in block["certifications"]
parts = split(cert, ":")
source = storage.pubkeys[parts[1]]
target = storage.pubkeys[parts[2]]
if (source, target) in keys(storage.cert_exp) # if cert previously existed (expiration planned)
if storage.cert_exp[(source, target)][end] < t # certification already expired
push!(storage.cert_in, (t, (source, target))) # add certification again
push!(storage.cert_exp[(source, target)], expiration) # plan new expiration
else # certification did not already expire (it's a renewal)
storage.cert_exp[(source, target)][end] = expiration # update last expiration date for this certification
end
else # new certification
push!(storage.cert_in, (t, (source, target))) # add certification
storage.cert_exp[(source, target)] = [expiration] # plan expiration
end
# Note: at the end, cert_exp contains a list of dates where the certification expired
end
end
end
"sample block every SAMPLING_PERIOD"
function get_sample(storage, block)
if block["number"] == 0
storage.firstblock_timestamp = block["time"]
end
sampled_blocks = length(storage.sampled_blocks) # number of steps already sampled
if block["time"] >= storage.firstblock_timestamp + sampled_blocks * SAMPLING_PERIOD
partialblock = PartialBlock(
block["version"],
block["number"],
block["powMin"],
block["time"],
block["medianTime"],
block["membersCount"],
length(storage.pseudos),
block["monetaryMass"],
block["issuersCount"],
block["issuersFrame"],
)
push!(storage.sampled_blocks, partialblock)
end
end
"parse blocks from chunks to get information into storage"
function parse_blocks!(storage::Storage)
storage_chunk = block_to_chunk(storage.lastblock)
last_local_chunk = local_chunk()
@showprogress "parsing chunks... " for chunknumber in storage_chunk : last_local_chunk
file = chunkfile(chunknumber)
s = String(read(file))
if CHUNK_FORMAT == "duniter"
blocks = JSON.parse(s)["blocks"] # duniter/bma format
elseif CHUNK_FORMAT == "dex"
blocks = JSON.parse(s) # dex format
else
error("please specify chunk format")
end
for block in blocks
if block["number"] <= storage.lastblock
continue # ignore blocks already taken into account
end
# @debug "adding $(block["number"])"
get_joiners(storage, block)
get_sample(storage, block)
get_leavers(storage, block)
get_new_certs(storage, block)
# (only used for last block)
storage.lastblock = block["number"]
storage.lastblock_timestamp = block["time"]
end
end
# fills in cert out from cert expiry
for (c,ts) in storage.cert_exp
for t in ts
if t <= storage.lastblock_timestamp
continue # only add cert expiry if not already in storage
end
push!(storage.cert_out, (t, c))
end
end
# sort it by time
sort!(storage.cert_out, by=x->x[1])
end
"browse blockchain from json chunk files and cache a Storage structure containing extracted data"
function browse()
storage = load_storage() # load last storage or create new
parse_blocks!(storage) # parse block not already in storage
serialize(STORAGE_CACHE_FILE, storage) # overwrites previous storage cache
end
end