从 ftp.ripe.net 下载 probes 存档,并使用 PHP 进行预处理:
<?php
set_time_limit(1800) ;
ini_set("memory_limit", "-1");
file_put_contents("last_run.log", date('Y-m-d H:i:s'));
function getFlags($code){
$code = strtoupper($code);
if($code == 'AD') return '??';
if($code == 'AE') return '??';
if($code == 'AF') return '??';
if($code == 'AG') return '??';
if($code == 'AI') return '??';
if($code == 'AL') return '??';
if($code == 'AM') return '??';
if($code == 'AO') return '??';
if($code == 'AQ') return '??';
if($code == 'AR') return '??';
if($code == 'AS') return '??';
if($code == 'AT') return '??';
if($code == 'AU') return '??';
if($code == 'AW') return '??';
if($code == 'AX') return '??';
if($code == 'AZ') return '??';
if($code == 'BA') return '??';
if($code == 'BB') return '??';
if($code == 'BD') return '??';
if($code == 'BE') return '??';
if($code == 'BF') return '??';
if($code == 'BG') return '??';
if($code == 'BH') return '??';
if($code == 'BI') return '??';
if($code == 'BJ') return '??';
if($code == 'BL') return '??';
if($code == 'BM') return '??';
if($code == 'BN') return '??';
if($code == 'BO') return '??';
if($code == 'BQ') return '??';
if($code == 'BR') return '??';
if($code == 'BS') return '??';
if($code == 'BT') return '??';
if($code == 'BV') return '??';
if($code == 'BW') return '??';
if($code == 'BY') return '??';
if($code == 'BZ') return '??';
if($code == 'CA') return '??';
if($code == 'CC') return '??';
if($code == 'CD') return '??';
if($code == 'CF') return '??';
if($code == 'CG') return '??';
if($code == 'CH') return '??';
if($code == 'CI') return '??';
if($code == 'CK') return '??';
if($code == 'CL') return '??';
if($code == 'CM') return '??';
if($code == 'CN') return '??';
if($code == 'CO') return '??';
if($code == 'CR') return '??';
if($code == 'CU') return '??';
if($code == 'CV') return '??';
if($code == 'CW') return '??';
if($code == 'CX') return '??';
if($code == 'CY') return '??';
if($code == 'CZ') return '??';
if($code == 'DE') return '??';
if($code == 'DJ') return '??';
if($code == 'DK') return '??';
if($code == 'DM') return '??';
if($code == 'DO') return '??';
if($code == 'DZ') return '??';
if($code == 'EC') return '??';
if($code == 'EE') return '??';
if($code == 'EG') return '??';
if($code == 'EH') return '??';
if($code == 'ER') return '??';
if($code == 'ES') return '??';
if($code == 'ET') return '??';
if($code == 'FI') return '??';
if($code == 'FJ') return '??';
if($code == 'FK') return '??';
if($code == 'FM') return '??';
if($code == 'FO') return '??';
if($code == 'FR') return '??';
if($code == 'GA') return '??';
if($code == 'GB') return '??';
if($code == 'GD') return '??';
if($code == 'GE') return '??';
if($code == 'GF') return '??';
if($code == 'GG') return '??';
if($code == 'GH') return '??';
if($code == 'GI') return '??';
if($code == 'GL') return '??';
if($code == 'GM') return '??';
if($code == 'GN') return '??';
if($code == 'GP') return '??';
if($code == 'GQ') return '??';
if($code == 'GR') return '??';
if($code == 'GS') return '??';
if($code == 'GT') return '??';
if($code == 'GU') return '??';
if($code == 'GW') return '??';
if($code == 'GY') return '??';
if($code == 'HK') return '??';
if($code == 'HM') return '??';
if($code == 'HN') return '??';
if($code == 'HR') return '??';
if($code == 'HT') return '??';
if($code == 'HU') return '??';
if($code == 'ID') return '??';
if($code == 'IE') return '??';
if($code == 'IL') return '??';
if($code == 'IM') return '??';
if($code == 'IN') return '??';
if($code == 'IO') return '??';
if($code == 'IQ') return '??';
if($code == 'IR') return '??';
if($code == 'IS') return '??';
if($code == 'IT') return '??';
if($code == 'JE') return '??';
if($code == 'JM') return '??';
if($code == 'JO') return '??';
if($code == 'JP') return '??';
if($code == 'KE') return '??';
if($code == 'KG') return '??';
if($code == 'KH') return '??';
if($code == 'KI') return '??';
if($code == 'KM') return '??';
if($code == 'KN') return '??';
if($code == 'KP') return '??';
if($code == 'KR') return '??';
if($code == 'KW') return '??';
if($code == 'KY') return '??';
if($code == 'KZ') return '??';
if($code == 'LA') return '??';
if($code == 'LB') return '??';
if($code == 'LC') return '??';
if($code == 'LI') return '??';
if($code == 'LK') return '??';
if($code == 'LR') return '??';
if($code == 'LS') return '??';
if($code == 'LT') return '??';
if($code == 'LU') return '??';
if($code == 'LV') return '??';
if($code == 'LY') return '??';
if($code == 'MA') return '??';
if($code == 'MC') return '??';
if($code == 'MD') return '??';
if($code == 'ME') return '??';
if($code == 'MF') return '??';
if($code == 'MG') return '??';
if($code == 'MH') return '??';
if($code == 'MK') return '??';
if($code == 'ML') return '??';
if($code == 'MM') return '??';
if($code == 'MN') return '??';
if($code == 'MO') return '??';
if($code == 'MP') return '??';
if($code == 'MQ') return '??';
if($code == 'MR') return '??';
if($code == 'MS') return '??';
if($code == 'MT') return '??';
if($code == 'MU') return '??';
if($code == 'MV') return '??';
if($code == 'MW') return '??';
if($code == 'MX') return '??';
if($code == 'MY') return '??';
if($code == 'MZ') return '??';
if($code == 'NA') return '??';
if($code == 'NC') return '??';
if($code == 'NE') return '??';
if($code == 'NF') return '??';
if($code == 'NG') return '??';
if($code == 'NI') return '??';
if($code == 'NL') return '??';
if($code == 'NO') return '??';
if($code == 'NP') return '??';
if($code == 'NR') return '??';
if($code == 'NU') return '??';
if($code == 'NZ') return '??';
if($code == 'OM') return '??';
if($code == 'PA') return '??';
if($code == 'PE') return '??';
if($code == 'PF') return '??';
if($code == 'PG') return '??';
if($code == 'PH') return '??';
if($code == 'PK') return '??';
if($code == 'PL') return '??';
if($code == 'PM') return '??';
if($code == 'PN') return '??';
if($code == 'PR') return '??';
if($code == 'PS') return '??';
if($code == 'PT') return '??';
if($code == 'PW') return '??';
if($code == 'PY') return '??';
if($code == 'QA') return '??';
if($code == 'RE') return '??';
if($code == 'RO') return '??';
if($code == 'RS') return '??';
if($code == 'RU') return '??';
if($code == 'RW') return '??';
if($code == 'SA') return '??';
if($code == 'SB') return '??';
if($code == 'SC') return '??';
if($code == 'SD') return '??';
if($code == 'SE') return '??';
if($code == 'SG') return '??';
if($code == 'SH') return '??';
if($code == 'SI') return '??';
if($code == 'SJ') return '??';
if($code == 'SK') return '??';
if($code == 'SL') return '??';
if($code == 'SM') return '??';
if($code == 'SN') return '??';
if($code == 'SO') return '??';
if($code == 'SR') return '??';
if($code == 'SS') return '??';
if($code == 'ST') return '??';
if($code == 'SV') return '??';
if($code == 'SX') return '??';
if($code == 'SY') return '??';
if($code == 'SZ') return '??';
if($code == 'TC') return '??';
if($code == 'TD') return '??';
if($code == 'TF') return '??';
if($code == 'TG') return '??';
if($code == 'TH') return '??';
if($code == 'TJ') return '??';
if($code == 'TK') return '??';
if($code == 'TL') return '??';
if($code == 'TM') return '??';
if($code == 'TN') return '??';
if($code == 'TO') return '??';
if($code == 'TR') return '??';
if($code == 'TT') return '??';
if($code == 'TV') return '??';
if($code == 'TW') return '??';
if($code == 'TZ') return '??';
if($code == 'UA') return '??';
if($code == 'UG') return '??';
if($code == 'UM') return '??';
if($code == 'US') return '??';
if($code == 'UY') return '??';
if($code == 'UZ') return '??';
if($code == 'VA') return '??';
if($code == 'VC') return '??';
if($code == 'VE') return '??';
if($code == 'VG') return '??';
if($code == 'VI') return '??';
if($code == 'VN') return '??';
if($code == 'VU') return '??';
if($code == 'WF') return '??';
if($code == 'WS') return '??';
if($code == 'XK') return '??';
if($code == 'YE') return '??';
if($code == 'YT') return '??';
if($code == 'ZA') return '??';
if($code == 'ZM') return '??';
return '?';
}
$path = date("Y/m",strtotime("-1 day"));
$fileName = date("Ymd",strtotime("-1 day")) . ".json.bz2";
$jsonFileName = date("Ymd",strtotime("-1 day")) . ".json";
$fileUrl = "https://ftp.ripe.net/ripe/atlas/probes/archive/$path/$fileName";
echo $fileUrl . "\n";
@unlink($jsonFileName);
@unlink($fileName);
echo "Retrieving http header...\n";
$header = get_headers($fileUrl);
$pp = "0";
echo json_encode($header, JSON_PRETTY_PRINT);
$key = key(preg_grep('/\bLength\b/i', $header));
$type = key(preg_grep('/\bType\b/i', $header));
$http = substr($header[0], 9, 3);
$tbytes = @explode(" ", $header[$key])[1];
$type = @explode("/", explode(" ", $header[$type])[1])[1];
echo " Target size: " . floor((($tbytes / 1000) / 1000)) . " Mo || " . floor(($tbytes / 1000)) . " Kb";
$remote = fopen($fileUrl, 'r');
$local = fopen($fileName, 'w');
$read_bytes = 0;
echo PHP_EOL;
while (!feof($remote)) {
$buffer = fread($remote, intval($tbytes));
fwrite($local, $buffer);
$read_bytes += 2048;
$progress = min(100, 100 * $read_bytes / $tbytes);
$progress = substr($progress, 0, 6) * 4;
$shell = 10; /* Progress bar width */
$rt = $shell * $progress / 100;
echo " \033[35;2m\e[0m Downloading: [" . round($progress, 3) . "%] " . floor((($read_bytes / 1000) * 4)) . "Kb ";
if ($pp === $shell) {
$pp = 0;
};
if ($rt === $shell) {
$rt = 0;
};
echo str_repeat("█", $rt) . str_repeat("=", ($pp++)) . ">@\r";
usleep(1000);
}
echo " \033[35;2m\e[0mDone [100%] " . floor((($tbytes / 1000) / 1000)) . " Mo || " . floor(($tbytes / 1000)) . " Kb \r";
echo PHP_EOL;
fclose($remote);
fclose($local);
echo 'bzip2 -d ' . $fileName . "\n";
system('bzip2 -d ' . $fileName);
echo "reading...\n";
$json = json_decode(file_get_contents($jsonFileName), true);
print_r($json['meta']);
@unlink($fileName);
@unlink($jsonFileName);
$probes = [];
echo "convert...\n";
foreach ($json['objects'] as &$item) {
foreach ($item as $key => $value) {
if(!in_array($key, ['id', 'address_v4', 'address_v6', 'asn_v4', 'asn_v6', 'country_code'])){
unset($item[$key]);
}
if($key == 'country_code') {
$item[$key] = getFlags($item[$key]) . " " . $item[$key];
}
}
$probes[$item['id']] = $item;
}
echo "write file: probes.json\n";
$json = json_encode($probes); //, JSON_PRETTY_PRINT);
file_put_contents("probes.json", $json);
echo "done.\n";
修改 ripe\atlas\tools\renderers\ping.py 文件,加入 JSON 解析:
# Copyright (c) 2016 RIPE NCC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# import sys
# reload(sys)
# sys.setdefaultencoding('utf-8')
import json
import io
from ..helpers.sanitisers import sanitise
from .base import Renderer as BaseRenderer
class Renderer(BaseRenderer):
"""
This is meant to be a stub example for what an aggregate renderer might look
like. If you have ideas as to how to make this better, feel free to send
along a pull request.
"""
RENDERS = [BaseRenderer.TYPE_PING]
def __init__(self, **kwargs):
BaseRenderer.__init__(self, **kwargs)
self.target = ""
self.packet_loss = 0
self.sent_packets = 0
self.received_packets = 0
self.rtts = []
self.rtts_min = []
self.rtts_max = []
self.rtt_types_map = {"min": self.rtts_min, "max": self.rtts_max}
def collect_stats(self, result):
"""
Calculates, stores and collects all stats we want from the given
result.
"""
if not self.target:
self.target = result.destination_name
self.sent_packets += result.packets_sent
self.received_packets += result.packets_received
self.collect_min_max_rtts("min", result.rtt_min)
self.collect_min_max_rtts("max", result.rtt_max)
self.collect_packets_rtt(result.packets)
def collect_min_max_rtts(self, rtt_type, rtt):
"""
Stores the given rtt in the corresponding list (min/max) if rtt is set.
"""
rtt = rtt
if not rtt:
rtt = 0
self.rtt_types_map[rtt_type].append(rtt)
def collect_packets_rtt(self, packets):
"""
Collects all the rrts of given packets and stores them
in our rtts list.
"""
for packet in packets:
rtt = packet.rtt
if not packet.rtt:
rtt = 0
self.rtts.append(rtt)
def calculate_loss(self):
"""Calculates the total loss between received and sent packets."""
if not self.sent_packets:
return 0
return (1 - float(self.received_packets) / self.sent_packets) * 100
def mean(self):
"""Calculates the mean of the collected rtts"""
return round(float(sum(self.rtts)) / max(len(self.rtts), 1), 3)
def median(self):
"""Calculates the median of the collected rtts"""
sorted_rtts = sorted(self.rtts)
index = (len(self.rtts) - 1) // 2
if len(self.rtts) % 2:
return sorted_rtts[index]
else:
return (sorted_rtts[index] + sorted_rtts[index + 1]) / 2.0
def on_result(self, result):
packets = result.packets
if not packets:
return "No packets found\n"
self.collect_stats(result)
# Because the origin value is more reliable as "from" in v4 and as
# "packet.source_address" in v6.
origin = result.origin
if ":" in origin:
origin = packets[0].source_address
times = " ".join(["{:11}".format(str(_.rtt) + " ms") for _ in packets])
f = io.open('/path/to/probes.json','r',encoding='utf-8')
probes = json.load(f)
return (
f"{result.packet_size} bytes from {result.destination_address} via "
f"probe #{result.probe_id:<7} {probes[str(result.probe_id)]['country_code']:<8}({origin:15})"
f": ttl={packets[0].ttl} times={times}\n"
)
def header(self, sample):
resolved_on = (
"server"
if sample.destination_address == sample.destination_name
else "probe"
)
return f"PING {sample.destination_name} (resolved on {resolved_on})\n"
def footer(self):
if not self.sent_packets:
return ""
self.packet_loss = self.calculate_loss()
return self.render_template(
"reports/aggregate_ping.txt",
target=sanitise(self.target),
sent=self.sent_packets,
received=self.received_packets,
packet_loss=self.packet_loss,
min=min(self.rtts_min),
median=self.median(),
mean=self.mean(),
max=max(self.rtts_max),
)
顺便说几句,RIPE 全网 Probes 30885 个,已连接 9347 个,可供 ICMP 测试的 6096 个。中国的一共 118 个,截止目前已连接的才 21 个。
可供 ICMP 测试(system-ipv4-works)稳定在线 1 天(system-ipv4-stable-1d 24*5%=1.2h/d)及以上的有 5 个,30 天及以上的只有 2 个(system-ipv4-stable-30d 30*24*5%=36h/m)。
关于 tags 描述可以看这里,国际互联网链接质量堪忧啊……?
根据朋友描述,申请硬件 Probe 不是很容易,而且因为是平邮,即使在地址上写明手机号,也不一定会收到……
但目前 RIPE 提供了软件 Probes,申请还是比较容易的,之前申请被拒的可以考虑申请软件版本。
软件 Probes 对 CentOS 7 提供了 rpm 的安装方式,其他系统需要自行编译安装或者使用 Docker 运行。
申请地址:
https://labs.ripe.net/Members/alun_davies/ripe-atlas-software-probes