streaming
streaming copied to clipboard
Add fine-grained timings to Writers
In streaming/base/format/base/writer.py
:
@classmethod
def _get_timer(cls) -> Timer:
"""Get a timer tree for the process of writing a dataset.
Returns:
Timer: The tree of timers.
"""
return Timer([
('write', Timer([
'serialize_sample',
'flush_shard',
])),
('finish',
Timer([
'flush_last_shard',
('save_index',
Timer([
'serialize_index',
'write_index',
'wait_for_shard_uploads',
'upload_index',
])),
'shutdown_executor',
'remove_local',
])),
])
Example output:
mds:
{
"named_timers": [
[
"write",
{
"named_timers": [
[
"serialize_sample",
{
"stats": {
"count": 33554432,
"frac_of_parts": 0.8812000213489666,
"frac_of_whole": 0.7268265642055434,
"max": 2.171626367,
"mean": 1.8077239529788458e-06,
"min": 1.276e-06,
"quantiles": [
1.276e-06,
1.457e-06,
1.49e-06,
1.517e-06,
1.544e-06,
1.573e-06,
1.606e-06,
1.65e-06,
1.722e-06,
2.131e-06,
2.171626367
],
"std": 0.00041819769639734895,
"total": 60.65715045499988
}
}
],
[
"flush_shard",
{
"stats": {
"count": 33554432,
"frac_of_parts": 0.11879997865103349,
"frac_of_whole": 0.09798794623091374,
"max": 0.018126742,
"mean": 2.4371035158634205e-07,
"min": 1.16e-07,
"quantiles": [
1.16e-07,
1.31e-07,
1.34e-07,
1.39e-07,
1.43e-07,
1.47e-07,
1.5e-07,
1.55e-07,
1.6e-07,
1.7e-07,
0.018126742
],
"std": 2.810335168383885e-05,
"total": 8.177562420000006
}
}
]
],
"stats": {
"count": 33554432,
"frac_of_parts": 0.9999638519330477,
"frac_of_whole": 0.8296981285330818,
"max": 2.171631721,
"mean": 2.4871462354362016e-06,
"min": 1.766e-06,
"quantiles": [
1.766e-06,
1.987e-06,
2.026e-06,
2.059e-06,
2.094e-06,
2.131e-06,
2.175e-06,
2.233e-06,
2.335e-06,
2.855e-06,
2.171631721
],
"std": 0.00041914432833075374,
"total": 83.45477923100002
}
}
],
[
"finish",
{
"named_timers": [
[
"flush_last_shard",
{
"stats": {
"frac_of_parts": 0.6023615745512895,
"frac_of_whole": 0.6012550226429129,
"total": 0.001813889
}
}
],
[
"save_index",
{
"named_timers": [
[
"serialize_index",
{
"stats": {
"frac_of_parts": 0.7763794708713782,
"frac_of_whole": 0.7714316449183856,
"total": 0.000824699
}
}
],
[
"write_index",
{
"stats": {
"frac_of_parts": 0.20490719114472572,
"frac_of_whole": 0.20360132828211963,
"total": 0.00021766
}
}
],
[
"wait_for_shard_uploads",
{
"stats": {
"frac_of_parts": 0.0009743588295267441,
"frac_of_whole": 0.0009681492914269679,
"total": 1.035e-06
}
}
],
[
"upload_index",
{
"stats": {
"frac_of_parts": 0.017738979154369505,
"frac_of_whole": 0.01762592956363126,
"total": 1.8843e-05
}
}
]
],
"stats": {
"frac_of_parts": 0.3550132567505818,
"frac_of_whole": 0.3543610893259764,
"total": 0.00106905
}
}
],
[
"shutdown_executor",
{
"stats": {
"frac_of_parts": 0.04247606346237633,
"frac_of_whole": 0.042398033968015524,
"total": 0.000127908
}
}
],
[
"remove_local",
{
"stats": {
"frac_of_parts": 0.00014910523575231395,
"frac_of_whole": 0.00014883132604402358,
"total": 4.49e-07
}
}
]
],
"stats": {
"frac_of_parts": 3.614806695237654e-05,
"frac_of_whole": 2.9993067691894386e-05,
"total": 0.003016838
}
}
]
],
"stats": {
"total": 100.584509427
}
}