Hi Peter,
For aggregating large volumes of data over large periods of time your best bet would be to use much smaller time windows (1 week like in the example below), and iterate over these time windows:
import time
import json
import urllib2
import urllib
import datetime
auth_token = "TOKEN"
api_url = "https://<your_domain>.wavefront.com"
TIME_WINDOW_DAYS = 7
start_ts = 1451635200 # 2016-01-01 00:00 -0700
QUERY = "100-ts(\"cpu.cpuidle\")"
output_file = open("query_output.txt", "w")
while start_ts + 60 * 60 * 24 * TIME_WINDOW_DAYS < time.time():
end_ts = start_ts + 60 * 60 * 24 * TIME_WINDOW_DAYS
print "Getting data from {} to {}".format(
datetime.datetime.fromtimestamp(start_ts).strftime('%Y-%m-%d %H:%M %z'),
datetime.datetime.fromtimestamp(end_ts).strftime('%Y-%m-%d %H:%M %z')
)
request_object = {"n": "query", "q": QUERY, "s": start_ts, "e": end_ts, "g": "h", "p": int(24 * TIME_WINDOW_DAYS),
"i": False, "autoEvents": False, "summarization": "MEAN", "strict": True}
req = urllib2.Request("{}/chart/api?{}".format(api_url, urllib.urlencode(request_object)))
req.add_header('X-AUTH-TOKEN', auth_token)
response = json.loads(urllib2.urlopen(req).read())
csv = []
for ts in response["timeseries"]:
base_row = ts['host'] + "," + ts['label']
if 'tags' in ts:
base_row += ",\"{}\"".format(" ".join(["[{}]=[{}]".format(tag, ts['tags'][tag]) for tag in ts['tags']]))
for d in ts['data']:
csv.append("{},{},{}".format(str(d[0]), str(d[1]), base_row))
output_file.write("\n".join(csv))
output_file.write("\n")
start_ts = end_ts
output_file.close()
print "Done!"
