give plugins control over step/heartbeat

- basic/fast plugins can use 10s
- more complex plugins (apache, mysql, bind, ..) will use 60s
- this is a little bit hackish since RRAs in configfile are based on the step value :)
This commit is contained in:
Olaf Rempel 2006-09-30 17:10:32 +02:00
parent a1c55be85e
commit bce2358271
18 changed files with 163 additions and 198 deletions

View File

@ -4,13 +4,14 @@
#include "configfile.h" #include "configfile.h"
#include "logging.h" #include "logging.h"
#include "plugins.h"
#define MAX_SIZE 8192 #define MAX_SIZE 8192
static char *pktbuf = NULL; static char *pktbuf = NULL;
static int pos = 0; static int pos = 0;
void net_submit(char *hostname, char *plugin, char *filename, int ds_id, char *data) void net_submit(char *hostname, struct sammler_plugin *plugin, char *filename, int ds_id, char *data)
{ {
int len = 0; int len = 0;
@ -28,7 +29,7 @@ void net_submit(char *hostname, char *plugin, char *filename, int ds_id, char *d
} }
len = snprintf(pktbuf + pos, MAX_SIZE - pos, "%s:%s:%d %s\n", len = snprintf(pktbuf + pos, MAX_SIZE - pos, "%s:%s:%d %s\n",
plugin, filename, ds_id, data); plugin->name, filename, ds_id, data);
if (len < 0 || len >= MAX_SIZE - pos) if (len < 0 || len >= MAX_SIZE - pos)
return; return;

View File

@ -1,7 +1,7 @@
#ifndef _NETWORK_H_ #ifndef _NETWORK_H_
#define _NETWORK_H_ #define _NETWORK_H_
void net_submit(char *hostname, char *plugin, char *filename, int ds_id, char *data); void net_submit(char *hostname, struct sammler_plugin *plugin, char *filename, int ds_id, char *data);
void net_commit(); void net_commit();
#endif /* _NETWORK_H_ */ #endif /* _NETWORK_H_ */

View File

@ -27,27 +27,26 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def[] = { static char *ds_def = {
"DS:entries:GAUGE:%d:0:U", "DS:entries:GAUGE:15:0:U "
"DS:searched:DERIVE:%d:0:U", "DS:searched:DERIVE:15:0:U "
"DS:found:DERIVE:%d:0:U", "DS:found:DERIVE:15:0:U "
"DS:new:DERIVE:%d:0:U", "DS:new:DERIVE:15:0:U "
"DS:invalid:DERIVE:%d:0:U", "DS:invalid:DERIVE:15:0:U "
"DS:ignore:DERIVE:%d:0:U", "DS:ignore:DERIVE:15:0:U "
"DS:delete:DERIVE:%d:0:U", "DS:delete:DERIVE:15:0:U "
"DS:delete_list:DERIVE:%d:0:U", "DS:delete_list:DERIVE:15:0:U "
"DS:insert:DERIVE:%d:0:U", "DS:insert:DERIVE:15:0:U "
"DS:insert_failed:DERIVE:%d:0:U", "DS:insert_failed:DERIVE:15:0:U "
"DS:drop:DERIVE:%d:0:U", "DS:drop:DERIVE:15:0:U "
"DS:early_drop:DERIVE:%d:0:U", "DS:early_drop:DERIVE:15:0:U "
"DS:icmp_error:DERIVE:%d:0:U", "DS:icmp_error:DERIVE:15:0:U "
"DS:expect_new:DERIVE:%d:0:U", "DS:expect_new:DERIVE:15:0:U "
"DS:expect_create:DERIVE:%d:0:U", "DS:expect_create:DERIVE:15:0:U "
"DS:expect_delete:DERIVE:%d:0:U", "DS:expect_delete:DERIVE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
return ds_def; return ds_def;
} }
@ -101,6 +100,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "ctstat", .name = "ctstat",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -24,14 +24,13 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def[] = { static char *ds_def = {
"DS:1min:GAUGE:%d:0:U", "DS:1min:GAUGE:15:0:U "
"DS:5min:GAUGE:%d:0:U", "DS:5min:GAUGE:15:0:U "
"DS:15min:GAUGE:%d:0:U", "DS:15min:GAUGE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
return ds_def; return ds_def;
} }
@ -64,6 +63,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "load", .name = "load",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -30,21 +30,19 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *mem_ds_def[] = { static char *mem_ds_def = {
"DS:total:GAUGE:%d:0:U", "DS:total:GAUGE:15:0:U "
"DS:free:GAUGE:%d:0:U", "DS:free:GAUGE:15:0:U "
"DS:buffers:GAUGE:%d:0:U", "DS:buffers:GAUGE:15:0:U "
"DS:cached:GAUGE:%d:0:U", "DS:cached:GAUGE:15:0:U "
NULL
}; };
static char *swap_ds_def[] = { static char *swap_ds_def = {
"DS:total:GAUGE:%d:0:U", "DS:total:GAUGE:15:0:U "
"DS:free:GAUGE:%d:0:U", "DS:free:GAUGE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
switch (ds_id) { switch (ds_id) {
case DS_MEMORY: case DS_MEMORY:
@ -119,6 +117,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "memory", .name = "memory",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -27,13 +27,12 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def[] = { static char *ds_def = {
"DS:block_total:GAUGE:%d:0:U", "DS:block_total:GAUGE:15:0:U "
"DS:block_free:GAUGE:%d:0:U", "DS:block_free:GAUGE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
return ds_def; return ds_def;
} }
@ -103,6 +102,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "mount", .name = "mount",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -27,15 +27,14 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def[] = { static char *ds_def = {
"DS:byte_in:COUNTER:%d:0:U", "DS:byte_in:COUNTER:15:0:U "
"DS:byte_out:COUNTER:%d:0:U", "DS:byte_out:COUNTER:15:0:U "
"DS:pkt_in:COUNTER:%d:0:U", "DS:pkt_in:COUNTER:15:0:U "
"DS:pkt_out:COUNTER:%d:0:U", "DS:pkt_out:COUNTER:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
return ds_def; return ds_def;
} }
@ -90,6 +89,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "netdev", .name = "netdev",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -24,12 +24,11 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def[] = { static char *ds_def = {
"DS:entropy:GAUGE:%d:0:U", "DS:entropy:GAUGE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
return ds_def; return ds_def;
} }
@ -62,6 +61,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "random", .name = "random",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -30,32 +30,30 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def_stat[] = { static char *ds_def_stat = {
"DS:in_hit:DERIVE:%d:0:U", "DS:in_hit:DERIVE:15:0:U "
"DS:in_slow_tot:DERIVE:%d:0:U", "DS:in_slow_tot:DERIVE:15:0:U "
"DS:in_slow_mc:DERIVE:%d:0:U", "DS:in_slow_mc:DERIVE:15:0:U "
"DS:in_no_route:DERIVE:%d:0:U", "DS:in_no_route:DERIVE:15:0:U "
"DS:in_brd:DERIVE:%d:0:U", "DS:in_brd:DERIVE:15:0:U "
"DS:in_martian_dst:DERIVE:%d:0:U", "DS:in_martian_dst:DERIVE:15:0:U "
"DS:in_martian_src:DERIVE:%d:0:U", "DS:in_martian_src:DERIVE:15:0:U "
"DS:out_hit:DERIVE:%d:0:U", "DS:out_hit:DERIVE:15:0:U "
"DS:out_slow_tot:DERIVE:%d:0:U", "DS:out_slow_tot:DERIVE:15:0:U "
"DS:out_slow_mc:DERIVE:%d:0:U", "DS:out_slow_mc:DERIVE:15:0:U "
"DS:in_hlist_search:DERIVE:%d:0:U", "DS:in_hlist_search:DERIVE:15:0:U "
"DS:out_hlist_search:DERIVE:%d:0:U", "DS:out_hlist_search:DERIVE:15:0:U "
NULL
}; };
static char *ds_def_gc[] = { static char *ds_def_gc = {
"DS:entries:GAUGE:%d:0:U", "DS:entries:GAUGE:15:0:U "
"DS:gc_total:DERIVE:%d:0:U", "DS:gc_total:DERIVE:15:0:U "
"DS:gc_ignored:DERIVE:%d:0:U", "DS:gc_ignored:DERIVE:15:0:U "
"DS:gc_goal_miss:DERIVE:%d:0:U", "DS:gc_goal_miss:DERIVE:15:0:U "
"DS:gc_dst_overflow:DERIVE:%d:0:U", "DS:gc_dst_overflow:DERIVE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
switch (ds_id) { switch (ds_id) {
case DS_STAT: case DS_STAT:
@ -125,6 +123,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "rtstat", .name = "rtstat",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -30,25 +30,23 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *cpu_ds_def[] = { static char *cpu_ds_def = {
"DS:user:COUNTER:%d:0:U", "DS:user:COUNTER:15:0:U "
"DS:nice:COUNTER:%d:0:U", "DS:nice:COUNTER:15:0:U "
"DS:syst:COUNTER:%d:0:U", "DS:syst:COUNTER:15:0:U "
"DS:idle:COUNTER:%d:0:U", "DS:idle:COUNTER:15:0:U "
"DS:wait:COUNTER:%d:0:U", "DS:wait:COUNTER:15:0:U "
"DS:intr:COUNTER:%d:0:U", "DS:intr:COUNTER:15:0:U "
"DS:sitr:COUNTER:%d:0:U", "DS:sitr:COUNTER:15:0:U "
NULL
}; };
static char *proc_ds_def[] = { static char *proc_ds_def = {
"DS:intr:COUNTER:%d:0:U", "DS:intr:COUNTER:15:0:U "
"DS:ctxt:COUNTER:%d:0:U", "DS:ctxt:COUNTER:15:0:U "
"DS:fork:COUNTER:%d:0:U", "DS:fork:COUNTER:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
switch (ds_id) { switch (ds_id) {
case DS_CPU: case DS_CPU:
@ -138,6 +136,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "stat", .name = "stat",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -24,13 +24,12 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def[] = { static char *ds_def = {
"DS:uptime:GAUGE:%d:0:U", "DS:uptime:GAUGE:15:0:U "
"DS:idletime:GAUGE:%d:0:U", "DS:idletime:GAUGE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
return ds_def; return ds_def;
} }
@ -63,6 +62,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "uptime", .name = "uptime",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -27,16 +27,15 @@
struct sammler_plugin plugin; struct sammler_plugin plugin;
static char *ds_def[] = { static char *ds_def = {
"DS:pgalloc_high:DERIVE:%d:0:U", "DS:pgalloc_high:DERIVE:15:0:U "
"DS:pgalloc_normal:DERIVE:%d:0:U", "DS:pgalloc_normal:DERIVE:15:0:U "
"DS:pgalloc_dma:DERIVE:%d:0:U", "DS:pgalloc_dma:DERIVE:15:0:U "
"DS:pgfree:DERIVE:%d:0:U", "DS:pgfree:DERIVE:15:0:U "
"DS:pgfault:DERIVE:%d:0:U", "DS:pgfault:DERIVE:15:0:U "
NULL
}; };
static char ** get_ds(int ds_id) static char * get_ds(int ds_id)
{ {
return ds_def; return ds_def;
} }
@ -95,6 +94,7 @@ static void probe(void)
struct sammler_plugin plugin = { struct sammler_plugin plugin = {
.name = "vmstat", .name = "vmstat",
.interval = 10,
.probe = &probe, .probe = &probe,
.get_ds = &get_ds, .get_ds = &get_ds,
}; };

View File

@ -23,6 +23,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdarg.h> #include <stdarg.h>
#include <time.h>
#include "list.h" #include "list.h"
@ -36,8 +37,6 @@
static LIST_HEAD(plugin_list); static LIST_HEAD(plugin_list);
static char *hostname = NULL;
static void plugin_load(char *filename) static void plugin_load(char *filename)
{ {
struct sammler_plugin *plugin = NULL; struct sammler_plugin *plugin = NULL;
@ -80,6 +79,7 @@ static void plugin_load(char *filename)
} }
log_print(LOG_INFO, "Plugin '%s' loaded", plugin->name); log_print(LOG_INFO, "Plugin '%s' loaded", plugin->name);
plugin->lastprobe = 0;
list_add_tail(&plugin->list, &plugin_list); list_add_tail(&plugin->list, &plugin_list);
@ -104,27 +104,24 @@ void plugin_load_all()
void plugins_probe(void) void plugins_probe(void)
{ {
struct sammler_plugin *plugin; struct sammler_plugin *plugin;
unsigned long now;
list_for_each_entry(plugin, &plugin_list, list) now = time(NULL);
list_for_each_entry(plugin, &plugin_list, list) {
if (plugin->lastprobe + plugin->interval <= now) {
plugin->probe(); plugin->probe();
plugin->lastprobe = now;
}
}
net_commit(); net_commit();
} }
char ** plugins_get_ds(char *name, int ds_id)
{
struct sammler_plugin *plugin;
list_for_each_entry(plugin, &plugin_list, list) {
if (strcmp(plugin->name, name))
continue;
return plugin->get_ds(ds_id);
}
return NULL;
}
void probe_submit(struct sammler_plugin *plugin, char *filename, int ds_id, const char *fmt, ... ) void probe_submit(struct sammler_plugin *plugin, char *filename, int ds_id, const char *fmt, ... )
{ {
static char *hostname = NULL;
va_list az; va_list az;
char *buffer; char *buffer;
int len; int len;
@ -148,8 +145,8 @@ void probe_submit(struct sammler_plugin *plugin, char *filename, int ds_id, cons
return; return;
} }
rrd_submit(hostname, plugin->name, filename, ds_id, buffer); rrd_submit(hostname, plugin, filename, ds_id, buffer);
net_submit(hostname, plugin->name, filename, ds_id, buffer); net_submit(hostname, plugin, filename, ds_id, buffer);
free(buffer); free(buffer);
} }

View File

@ -7,8 +7,10 @@
struct sammler_plugin { struct sammler_plugin {
struct list_head list; struct list_head list;
char *name; char *name;
unsigned int interval;
unsigned long lastprobe;
void (*probe) (void); void (*probe) (void);
char ** (*get_ds) (int ds_id); char * (*get_ds) (int ds_id);
}; };
void plugin_load_all(void); void plugin_load_all(void);

View File

@ -71,41 +71,6 @@ static int append_rra_config(char *buffer, int size, int *pos)
return rra_cnt; return rra_cnt;
} }
static int append_ds_config(char *buffer, int size, int *pos, int heartbeat, char **ds_def)
{
int len, ds_cnt = 0;
char *dsbuild;
dsbuild = malloc(BUFSIZE);
if (dsbuild == NULL) {
log_print(LOG_ERROR, "append_ds_config: out of memory");
return -1;
}
while (*ds_def != NULL) {
len = snprintf(dsbuild, BUFSIZE, *ds_def, heartbeat);
if (len < 0 || len >= BUFSIZE) {
log_print(LOG_ERROR, "append_ds_config: arguments too long");
free(dsbuild);
return -1;
}
len = snprintf(buffer + *pos, size - *pos, "%s ", dsbuild);
if (len < 0 || len >= size - *pos) {
log_print(LOG_ERROR, "append_ds_config: arguments too long");
free(dsbuild);
return -1;
}
*pos += len;
*ds_def++;
ds_cnt++;
}
free(dsbuild);
return ds_cnt;
}
static int do_rrd(int mode, char *cmd) static int do_rrd(int mode, char *cmd)
{ {
int argc; int argc;
@ -136,10 +101,16 @@ static int do_rrd(int mode, char *cmd)
return 0; return 0;
} }
static int rrd_create_file(char *filename, char **ds_def) static int rrd_create_file(char *filename, struct sammler_plugin *plugin, int ds_id)
{ {
int pos, step, heartbeat, retval; int pos, step, retval;
char *buffer; char *ds_def, *buffer;
ds_def = plugin->get_ds(ds_id);
if (ds_def == NULL) {
log_print(LOG_ERROR, "No vaild DS found (%s:%d)", plugin->name, ds_id);
return -1;
}
buffer = malloc(ARGVSIZE); buffer = malloc(ARGVSIZE);
if (buffer == NULL) { if (buffer == NULL) {
@ -147,10 +118,9 @@ static int rrd_create_file(char *filename, char **ds_def)
return -1; return -1;
} }
step = config_get_int("global", "step", DEFAULT_STEP); step = plugin->interval;
heartbeat = (step * 2) + (step / 2);
pos = snprintf(buffer, ARGVSIZE, "create %s -s %d ", filename, step); pos = snprintf(buffer, ARGVSIZE, "create %s -s %d %s ", filename, step, ds_def);
if (pos < 0 || pos >= ARGVSIZE) { if (pos < 0 || pos >= ARGVSIZE) {
log_print(LOG_ERROR, "rrd_create_file: arguments too long"); log_print(LOG_ERROR, "rrd_create_file: arguments too long");
free(buffer); free(buffer);
@ -162,11 +132,6 @@ static int rrd_create_file(char *filename, char **ds_def)
return -1; return -1;
} }
if (append_ds_config(buffer, ARGVSIZE, &pos, heartbeat, ds_def) <= 0) {
free(buffer);
return -1;
}
retval = do_rrd(RRDTOOL_CREATE, buffer); retval = do_rrd(RRDTOOL_CREATE, buffer);
free(buffer); free(buffer);
@ -254,12 +219,12 @@ static int create_parent_dirs(char *filename)
return 0; return 0;
} }
void rrd_submit(char *hostname, char *plugin, char *filename, int ds_id, char *data) void rrd_submit(char *hostname, struct sammler_plugin *plugin, char *filename, int ds_id, char *data)
{ {
struct stat statbuf; struct stat statbuf;
static char *rrd_dir = NULL; static char *rrd_dir = NULL;
char *fullfile, **ds_def; char *fullfile;
int len; int len;
if (rrd_dir == NULL) if (rrd_dir == NULL)
@ -287,14 +252,7 @@ void rrd_submit(char *hostname, char *plugin, char *filename, int ds_id, char *d
return; return;
} }
ds_def = plugins_get_ds(plugin, ds_id); if (rrd_create_file(fullfile, plugin, ds_id) == -1) {
if (ds_def == NULL) {
log_print(LOG_ERROR, "No vaild DS found (%s:%d)", plugin, ds_id);
free(fullfile);
return;
}
if (rrd_create_file(fullfile, ds_def) == -1) {
free(fullfile); free(fullfile);
return; return;
} }

View File

@ -1,6 +1,6 @@
#ifndef _RRDTOOL_H_ #ifndef _RRDTOOL_H_
#define _RRDTOOL_H_ #define _RRDTOOL_H_
void rrd_submit(char *hostname, char *plugin, char *filename, int ds_id, char *data); void rrd_submit(char *hostname, struct sammler_plugin *plugin, char *filename, int ds_id, char *data);
#endif /* _RRDTOOL_H_ */ #endif /* _RRDTOOL_H_ */

View File

@ -23,6 +23,9 @@
#include <string.h> #include <string.h>
#include <getopt.h> #include <getopt.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <signal.h>
#include "configfile.h" #include "configfile.h"
#include "logging.h" #include "logging.h"
@ -94,9 +97,26 @@ int main(int argc, char *argv[])
plugin_load_all(); plugin_load_all();
fd_set fdsel, fdcpy;
FD_ZERO(&fdsel);
struct timeval tv;
tv.tv_sec = 1;
tv.tv_usec = 0;
while (1) { while (1) {
memcpy(&fdcpy, &fdsel, sizeof(fdsel));
int i = select(64, &fdcpy, NULL, NULL, &tv);
if (i == -1) {
log_print(LOG_ERROR, "select()");
/* timeout */
} else if (i == 0) {
plugins_probe(); plugins_probe();
sleep (10); tv.tv_sec = 1;
tv.tv_usec = 0;
}
} }
return 0; return 0;

View File

@ -3,7 +3,6 @@ hostname localhost
logfile sammler.log logfile sammler.log
step 10
rrd_dir rrd rrd_dir rrd
plugin_dir . plugin_dir .
@ -18,18 +17,9 @@ plugin p_mount.so
#plugin p_rtstat.so #plugin p_rtstat.so
plugin p_random.so plugin p_random.so
# for basic plugins (interval 10s)
# 1h(10s), 48h(1min), 7d(5min), 4w(30min) # 1h(10s), 48h(1min), 7d(5min), 4w(30min)
rra RRA:MIN:0.1:1:360 rra RRA:MIN:0.1:1:360 RRA:AVERAGE:0.1:1:360 RRA:MAX:0.1:1:360
rra RRA:MIN:0.1:6:2880 rra RRA:MIN:0.1:6:2880 RRA:AVERAGE:0.1:6:2880 RRA:MAX:0.1:6:2880
rra RRA:MIN:0.1:30:2016 rra RRA:MIN:0.1:30:2016 RRA:AVERAGE:0.1:30:2016 RRA:MAX:0.1:30:2016
rra RRA:MIN:0.1:180:1344 rra RRA:MIN:0.1:180:1344 RRA:AVERAGE:0.1:180:1344 RRA:MAX:0.1:180:1344
rra RRA:AVERAGE:0.1:1:360
rra RRA:AVERAGE:0.1:6:2880
rra RRA:AVERAGE:0.1:30:2016
rra RRA:AVERAGE:0.1:180:1344
rra RRA:MAX:0.1:1:360
rra RRA:MAX:0.1:6:2880
rra RRA:MAX:0.1:30:2016
rra RRA:MAX:0.1:180:1344