1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

use super::{
    ensure_nodes_are_in_sync, MeasurementReportInterval, MeasurementReporter, SyncNode,
    SyncWaitParams,
};
use crate::{
    jormungandr::FragmentNode,
    testing::{benchmark_efficiency, benchmark_speed, Speed, Thresholds, VerificationError},
};
use chain_impl_mockchain::fragment::FragmentId;
use std::time::{Duration, SystemTime};

pub fn measure_how_many_nodes_are_running<A: SyncNode + ?Sized>(leaders: &[&A], name: &str) {
    let leaders_nodes_count = leaders.len() as u32;

    let mut efficiency_benchmark_run = benchmark_efficiency(name)
        .target(leaders_nodes_count)
        .start();
    let mut leaders_ids: Vec<u32> = (1..=leaders_nodes_count).collect();
    let now = SystemTime::now();

    loop {
        if now.elapsed().unwrap().as_secs() > (10 * 60) {
            break;
        }
        std::thread::sleep(Duration::from_secs(10));

        leaders_ids.retain(|leader_id| {
            let leader_index_usize = (leader_id - 1) as usize;
            let leader: &A = leaders.get(leader_index_usize).unwrap();
            if leader.is_running() {
                efficiency_benchmark_run.increment();
                return false;
            }
            true
        });

        if leaders_ids.is_empty() {
            break;
        }
    }

    print_error_for_failed_leaders(leaders_ids, leaders);

    efficiency_benchmark_run.stop().print()
}

fn print_error_for_failed_leaders<A: SyncNode + ?Sized>(leaders_ids: Vec<u32>, leaders: &[&A]) {
    if leaders_ids.is_empty() {
        return;
    }

    println!("Nodes which failed to bootstrap: ");
    for leader_id in leaders_ids {
        let leader_index_usize = (leader_id - 1) as usize;
        let leader = leaders.get(leader_index_usize).unwrap();
        println!(
            "{} - Error Logs: {:?}",
            leader.alias(),
            leader.get_lines_with_error_and_invalid()
        );
    }
}

pub fn measure_fragment_propagation_speed<A: FragmentNode + Sized>(
    fragment_id: FragmentId,
    leaders: &[&A],
    sync_wait: Thresholds<Speed>,
    info: &str,
    report_node_stats_interval: MeasurementReportInterval,
) -> Result<(), VerificationError> {
    let benchmark = benchmark_speed(info.to_owned())
        .with_thresholds(sync_wait)
        .start();

    let leaders_nodes_count = leaders.len() as u32;
    let mut report_node_stats = MeasurementReporter::new(report_node_stats_interval);
    let mut leaders_ids: Vec<u32> = (1..=leaders_nodes_count).collect();

    while !benchmark.timeout_exceeded() {
        leaders_ids.retain(|leader_id| {
            let leader_index_usize = (leader_id - 1) as usize;
            let leader: &A = leaders.get(leader_index_usize).unwrap();
            let fragment_logs = leader.fragment_logs().unwrap();
            let alias = FragmentNode::alias(leader);
            report_node_stats
                .do_if_interval_reached(|| println!("Node: {} -> {:?}", alias, fragment_logs));

            !fragment_logs.iter().any(|(id, _)| *id == fragment_id)
        });
        report_node_stats.increment();

        if leaders_ids.is_empty() {
            benchmark.stop().print();
            break;
        }
    }
    Ok(())
}

pub fn measure_and_log_sync_time<A: SyncNode + ?Sized>(
    nodes: &[&A],
    sync_wait: Thresholds<Speed>,
    info: &str,
    report_node_stats_interval: MeasurementReportInterval,
) -> Result<(), VerificationError> {
    let benchmark = benchmark_speed(info.to_owned())
        .with_thresholds(sync_wait)
        .start();

    let mut report_node_stats_counter = 0u32;
    let interval: u32 = report_node_stats_interval.into();

    while !benchmark.timeout_exceeded() {
        let tips = nodes
            .iter()
            .map(|node| {
                if report_node_stats_counter >= interval {
                    node.log_stats();
                }
                node.tip()
            })
            .collect::<Vec<_>>();

        if report_node_stats_counter >= interval {
            println!("Measuring sync time... current block tips: {:?}", tips);
            report_node_stats_counter = 0;
        } else {
            report_node_stats_counter += 1;
        }

        let first = tips.first().cloned();
        let stop = first
            .map(|tip| tips.into_iter().all(|t| t == tip))
            .unwrap_or(true);

        if stop {
            benchmark.stop().print();
            return Ok(());
        }
    }

    // we know it fails, this method is used only for reporting
    let result = ensure_nodes_are_in_sync(SyncWaitParams::ZeroWait, nodes);
    benchmark.stop().print();
    result
}