Skip to content

Commit

Permalink
Report timers/histograms, add hyphen prefix support (#8)
Browse files Browse the repository at this point in the history 
In existing postmates systems there is a need to parse metrics with
names like

    source.machine-metric.name

as being a metric named 'metric.name' with source 'source.machine'.
The etsy/statsd backend has been modified to take a regex to fiddle
with this but I've hard coded it in the expectation that future
backends will be programmable by the end-user.

This commit:

  * Fixes #4
  * Fixes #7

Signed-off-by: Brian L. Troutwine <blt@postmates.com>
  • Loading branch information
@blt
blt committed Jun 7, 2016
1 parent f3cb75d commit 28c629e
Show file tree
Hide file tree
Showing 10 changed files with 401 additions and 50 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ mime = "0.2.0"
rustc-serialize = "0.3.19"
time = "0.1"
url = "1.1.1"
regex = "0.1"

[profile.release]
lto = true
6 changes: 4 additions & 2 deletions src/backend.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ pub fn factory(console: &bool,
metric_source)));
}
if *librato {
backends.push(Box::new(librato::Librato::new(librato_username, librato_token,
metric_source, librato_host)));
backends.push(Box::new(librato::Librato::new(librato_username,
librato_token,
metric_source,
librato_host)));
}
backends.into_boxed_slice()
}
Expand Down
193 changes: 181 additions & 12 deletions src/backends/librato.rs
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
use super::super::backend::Backend;
use super::super::buckets::Buckets;
use std::str::FromStr;
use std::collections::BTreeMap;
use time;
use rustc_serialize::json;
use rustc_serialize::json::{Json, ToJson};
use hyper::client::Client;
use hyper::header::{ContentType, Authorization, Basic, Connection};
use url;
use mime::Mime;
use regex::Regex;

#[derive(Debug)]
pub struct Librato {
Expand All @@ -16,26 +18,70 @@ pub struct Librato {
host: String,
}

#[derive(RustcDecodable, RustcEncodable, Debug)]
#[derive(Debug)]
pub struct LCounter {
name: String,
value: f64,
source: Option<String>,
}

#[derive(RustcDecodable, RustcEncodable, Debug)]
impl ToJson for LCounter {
fn to_json(&self) -> Json {
let mut d = BTreeMap::new();
d.insert("name".to_string(), self.name.to_json());
d.insert("value".to_string(), self.value.to_json());
match self.source {
Some(ref src) => {
d.insert("source".to_string(), src.to_json());
()
}
None => (),
};
Json::Object(d)
}
}

#[derive(Debug)]
pub struct LGuage {
name: String,
value: f64,
source: Option<String>,
}

impl ToJson for LGuage {
fn to_json(&self) -> Json {
let mut d = BTreeMap::new();
d.insert("name".to_string(), self.name.to_json());
d.insert("value".to_string(), self.value.to_json());
match self.source {
Some(ref src) => {
d.insert("source".to_string(), src.to_json());
()
}
None => (),
};
Json::Object(d)
}
}

#[derive(RustcDecodable, RustcEncodable, Debug)]
#[derive(Debug)]
pub struct LPayload {
guages: Vec<LGuage>,
counters: Vec<LCounter>,
source: String,
measure_time: i64,
}

impl ToJson for LPayload {
fn to_json(&self) -> Json {
let mut d = BTreeMap::new();
d.insert("guages".to_string(), self.guages.to_json());
d.insert("counters".to_string(), self.counters.to_json());
d.insert("source".to_string(), self.source.to_json());
d.insert("measure_time".to_string(), self.measure_time.to_json());
Json::Object(d)
}
}

impl Librato {
/// Create a Librato formatter
Expand All @@ -62,37 +108,128 @@ impl Librato {
None => time::get_time().sec,
};

let re = Regex::new(r"(?x)
((?P<source>.*)-)? # the source
(?P<metric>.*) # the metric
")
.unwrap();

let mut guages = vec![];
let mut counters = vec![];

counters.push(LCounter {
name: "cernan.bad_messages".to_string(),
value: buckets.bad_messages() as f64,
source: None,
});
counters.push(LCounter {
name: "cernan.total_messages".to_string(),
value: buckets.total_messages() as f64,
source: None,
});

for (key, value) in buckets.counters().iter() {
let caps = re.captures(&key).unwrap();
counters.push(LCounter {
name: key.to_string(),
name: caps.name("metric").unwrap().to_string(),
value: *value,
source: caps.name("source").map(|x| x.to_string()),
});
}
for (key, value) in buckets.gauges().iter() {
let caps = re.captures(&key).unwrap();
guages.push(LGuage {
name: key.to_string(),
name: caps.name("metric").unwrap().to_string(),
value: *value,
source: caps.name("source").map(|x| x.to_string()),
});
}

for (key, value) in buckets.histograms().iter() {
let caps = re.captures(&key).unwrap();
guages.push(LGuage {
name: format!("{}.min", caps.name("metric").unwrap().to_string()),
value: value.min().unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.max", caps.name("metric").unwrap().to_string()),
value: value.max().unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.mean", caps.name("metric").unwrap().to_string()),
value: value.mean().unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.50", caps.name("metric").unwrap().to_string()),
value: value.percentile(50.0).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.90", caps.name("metric").unwrap().to_string()),
value: value.percentile(90.0).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.99", caps.name("metric").unwrap().to_string()),
value: value.percentile(99.0).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.999", caps.name("metric").unwrap().to_string()),
value: value.percentile(99.9).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
}

for (key, value) in buckets.timers().iter() {
let caps = re.captures(&key).unwrap();
guages.push(LGuage {
name: format!("{}.min", caps.name("metric").unwrap().to_string()),
value: value.min().unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.max", caps.name("metric").unwrap().to_string()),
value: value.max().unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.mean", caps.name("metric").unwrap().to_string()),
value: value.mean().unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.50", caps.name("metric").unwrap().to_string()),
value: value.percentile(50.0).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.90", caps.name("metric").unwrap().to_string()),
value: value.percentile(90.0).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.99", caps.name("metric").unwrap().to_string()),
value: value.percentile(99.0).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
guages.push(LGuage {
name: format!("{}.999", caps.name("metric").unwrap().to_string()),
value: value.percentile(99.9).unwrap(),
source: caps.name("source").map(|x| x.to_string()),
});
}

let obj = LPayload {
guages: guages,
counters: counters,
source: self.source.clone(),
measure_time: start,
};
json::encode(&obj).unwrap()
obj.to_json().to_string()
}
}

Expand Down Expand Up @@ -121,12 +258,14 @@ impl Backend for Librato {
mod test {
use super::super::super::metric::{Metric, MetricKind};
use super::super::super::buckets::Buckets;
use regex::Regex;
use super::*;

fn make_buckets() -> Buckets {
let mut buckets = Buckets::new();
let m1 = Metric::new("test.counter", 1.0, MetricKind::Counter(1.0));
let m2 = Metric::new("test.gauge", 3.211, MetricKind::Gauge);
let m6 = Metric::new("src-test.gauge.2", 3.211, MetricKind::Gauge);

let m3 = Metric::new("test.timer", 12.101, MetricKind::Timer);
let m4 = Metric::new("test.timer", 1.101, MetricKind::Timer);
Expand All @@ -136,19 +275,49 @@ mod test {
buckets.add(&m3);
buckets.add(&m4);
buckets.add(&m5);
buckets.add(&m6);
buckets
}

#[test]
fn test_our_regex_with_source() {
let re = Regex::new(r"(?x)
((?P<source>.*)-)? # the source
(?P<metric>.*) # the metric
")
.unwrap();
let caps = re.captures("source-foo.bar.baz").unwrap();
assert_eq!(Some("source"), caps.name("source"));
assert_eq!(Some("foo.bar.baz"), caps.name("metric"));
}

#[test]
fn test_our_regex_no_source() {
let re = Regex::new(r"(?x)
((?P<source>.*)-)? # the source
(?P<metric>.*) # the metric
")
.unwrap();
let caps = re.captures("foo.bar.baz").unwrap();
assert_eq!(None, caps.name("source"));
assert_eq!(Some("foo.bar.baz"), caps.name("metric"));
}

#[test]
fn test_format_librato_buckets_no_timers() {
let buckets = make_buckets();
let librato = Librato::new("user", "token", "test-src", "http://librato.example.com");
let result = librato.format_stats(&buckets, Some(10101));

assert!(result ==
"{\"guages\":[{\"name\":\"test.gauge\",\"value\":3.211}],\"counters\":[{\"name\":\
\"cernan.bad_messages\",\"value\":0.0},{\"name\":\"cernan.total_messages\",\
\"value\":5.0},{\"name\":\"test.counter\",\"value\":1.0}],\"source\":\
\"test-src\",\"measure_time\":10101}");
assert_eq!("{\"counters\":[{\"name\":\"cernan.bad_messages\",\"value\":0.0},{\"name\":\
\"cernan.total_messages\",\"value\":6.0},{\"name\":\"test.counter\",\
\"value\":1.0}],\"guages\":[{\"name\":\"test.gauge\",\"value\":3.211},\
{\"name\":\"test.gauge.2\",\"source\":\"src\",\"value\":3.211},{\"name\":\
\"test.timer.min\",\"value\":1.1},{\"name\":\"test.timer.max\",\"value\":12.\
1},{\"name\":\"test.timer.mean\",\"value\":5.43},{\"name\":\"test.timer.50\",\
\"value\":12.1},{\"name\":\"test.timer.90\",\"value\":12.1},{\"name\":\"test.\
timer.99\",\"value\":12.1},{\"name\":\"test.timer.999\",\"value\":12.1}],\
\"measure_time\":10101,\"source\":\"test-src\"}",
result);
}
}
Loading

0 comments on commit 28c629e

Please sign in to comment.